You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by co...@apache.org on 2026/10/10 11:28:38 UTC
(tsfile) 03/08: fix(cpp): preserve quoted CSV line endings in tsfile-cli (#996)
This is an automated email from the ASF dual-hosted git repository.
ColinLeeo pushed a commit to branch rc/2.5.0
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 2e3ff3bb301feb956d6af79646823bfbb256ffe5
Author: Colin Lee <10...@users.noreply.github.com>
AuthorDate: Sat Oct 10 10:43:37 2026 +0800
fix(cpp): preserve quoted CSV line endings in tsfile-cli (#996)
* fix(cpp): preserve quoted CSV line endings in tsfile-cli
* test(cpp): write embedded-newline CSV fixture in binary mode
---
cpp/test/tools/command_e2e_test.cc | 90 ++++++++++++++++++++++++++++++++++++-
cpp/test/tools/input_format_test.cc | 38 +++++++++++++++-
cpp/tools/commands/cmd_write.cc | 2 +-
cpp/tools/format/input_format.cc | 24 +++++-----
cpp/tools/format/input_format.h | 12 ++---
cpp/tools/tools_main.cc | 21 ++++++++-
6 files changed, 166 insertions(+), 21 deletions(-)
diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc
index b231bdac5..e9f8cde71 100644
--- a/cpp/test/tools/command_e2e_test.cc
+++ b/cpp/test/tools/command_e2e_test.cc
@@ -849,7 +849,8 @@ TEST(CliE2E, WriteImportsQuotedFieldWithEmbeddedNewline) {
std::string csv_path =
tsfile_cli_test::unique_temp_path("tsfile_cli_nl_in", ".csv");
{
- std::ofstream o(csv_path.c_str());
+ // Keep the fixture's LF bytes unchanged on Windows as well.
+ std::ofstream o(csv_path.c_str(), std::ios::binary);
// The note field on the first row spans two physical lines inside
// quotes; it must import as a single row, not be split into two.
o << "time,id1,note\n0,dev,\"line one\nline two\"\n1,dev,plain\n";
@@ -1985,3 +1986,90 @@ TEST(CliE2E, WriteCatCsvWritePreservesLeadingBackslashes) {
std::remove(exported.c_str());
std::remove(second.c_str());
}
+
+TEST(CliE2E, WriteCatAndExportCsvPreserveQuotedLineEndings) {
+ const std::string input =
+ tsfile_cli_test::unique_temp_path("tsfile_cli_crlf_input", ".csv");
+ const std::string first =
+ tsfile_cli_test::unique_temp_path("tsfile_cli_crlf_first", ".tsfile");
+ const std::string expected_csv =
+ "time,site,message,note\n"
+ "0,\"s\r\n1\",\"first\r\nsecond\",\"\"\"quoted\"\"\r\n\r\nlast\"\n"
+ "1,\"s\r\n1\",\"lf\nonly\",\"mixed\r\nand\nmore\"\n"
+ "2,\"s\r\n1\",\"carriage\rreturn\",\"\r\n\"\n"
+ "3,\"s\r\n1\",\"ends\r\",\"line\r\nend\"\n";
+ {
+ std::ofstream csv(input.c_str(), std::ios::binary);
+ ASSERT_TRUE(csv.good());
+ // CRLF record boundaries, embedded CRLF/LF/CR, and no final newline.
+ csv << "time,site,message,note\r\n"
+ "0,\"s\r\n1\",\"first\r\nsecond\","
+ "\"\"\"quoted\"\"\r\n\r\nlast\"\r\n"
+ "1,\"s\r\n1\",\"lf\nonly\",\"mixed\r\nand\nmore\"\r\n"
+ "2,\"s\r\n1\",\"carriage\rreturn\",\"\r\n\"\r\n"
+ "3,\"s\r\n1\",\"ends\r\",\"line\r\nend\"";
+ }
+ const auto write = [&](const std::string& source,
+ const std::string& target) {
+ std::ostringstream out, err;
+ const int code = tsfile_cli::run_cli(
+ {"write", "--table", "t", "--tag", "site", "STRING", "--field",
+ "message", "STRING", "--field", "note", "TEXT", "-i", source, "-o",
+ target},
+ out, err);
+ EXPECT_EQ(code, 0) << err.str();
+ return code;
+ };
+ const auto read = [&](const std::string& path, const std::string& format) {
+ std::ostringstream out, err;
+ EXPECT_EQ(tsfile_cli::run_cli({"cat", "-t", "t", "-f", format, path},
+ out, err),
+ 0)
+ << err.str();
+ return out.str();
+ };
+ const std::string expected_ndjson =
+ R"json({"time":"0","site":"s\r\n1","message":"first\r\nsecond","note":"\"quoted\"\r\n\r\nlast"})json"
+ "\n"
+ R"json({"time":"1","site":"s\r\n1","message":"lf\nonly","note":"mixed\r\nand\nmore"})json"
+ "\n"
+ R"json({"time":"2","site":"s\r\n1","message":"carriage\rreturn","note":"\r\n"})json"
+ "\n"
+ R"json({"time":"3","site":"s\r\n1","message":"ends\r","note":"line\r\nend"})json"
+ "\n";
+ ASSERT_EQ(write(input, first), 0);
+ EXPECT_EQ(read(first, "ndjson"), expected_ndjson);
+
+ for (const std::string command : {"cat", "export"}) {
+ SCOPED_TRACE(command);
+ const std::string exported = tsfile_cli_test::unique_temp_path(
+ "tsfile_cli_crlf_" + command, ".csv");
+ const std::string second = tsfile_cli_test::unique_temp_path(
+ "tsfile_cli_crlf_" + command, ".tsfile");
+ if (command == "cat") {
+ const std::string csv = read(first, "csv");
+ EXPECT_EQ(csv, expected_csv);
+ std::ofstream file(exported.c_str(), std::ios::binary);
+ ASSERT_TRUE(file.good());
+ file << csv;
+ } else {
+ std::ostringstream out, err;
+ ASSERT_EQ(tsfile_cli::run_cli({"export", "-t", "t", "--type", "csv",
+ "-o", exported, first},
+ out, err),
+ 0)
+ << err.str();
+ std::ifstream file(exported.c_str(), std::ios::binary);
+ ASSERT_TRUE(file.good());
+ std::ostringstream bytes;
+ bytes << file.rdbuf();
+ EXPECT_EQ(bytes.str(), expected_csv);
+ }
+ ASSERT_EQ(write(exported, second), 0);
+ EXPECT_EQ(read(second, "ndjson"), expected_ndjson);
+ std::remove(exported.c_str());
+ std::remove(second.c_str());
+ }
+ std::remove(input.c_str());
+ std::remove(first.c_str());
+}
diff --git a/cpp/test/tools/input_format_test.cc b/cpp/test/tools/input_format_test.cc
index ba922ebc1..8e6707973 100644
--- a/cpp/test/tools/input_format_test.cc
+++ b/cpp/test/tools/input_format_test.cc
@@ -158,16 +158,52 @@ TEST(InputFormatTest, ReadRecordStripsCarriageReturn) {
long long n = 0;
ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n));
EXPECT_EQ(rec, "a,b");
+ EXPECT_EQ(n, 1);
+ ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n));
+ EXPECT_EQ(rec, "c,d");
+ EXPECT_EQ(n, 1);
+ EXPECT_FALSE(tsfile_cli::read_record(in, true, rec, n));
+}
+
+TEST(InputFormatTest, ReadRecordPreservesQuotedLineEndings) {
+ std::istringstream in(
+ "1,\"line one\r\n\r\nline \"\"two\"\"\nline three\",x\r\n"
+ "2,plain,y\r\n");
+ std::string rec;
+ long long n = 0;
+ ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n));
+ EXPECT_EQ(rec, "1,\"line one\r\n\r\nline \"\"two\"\"\nline three\",x");
+ EXPECT_EQ(n, 4);
+ const auto fields = tsfile_cli::split_line(rec, ',', true);
+ ASSERT_EQ(fields.size(), 3u);
+ EXPECT_EQ(fields[1], "line one\r\n\r\nline \"two\"\nline three");
+ ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n));
+ EXPECT_EQ(rec, "2,plain,y");
+ EXPECT_EQ(n, 1);
+ EXPECT_FALSE(tsfile_cli::read_record(in, true, rec, n));
+}
+
+TEST(InputFormatTest, ReadRecordPreservesQuotedCarriageReturnAtEof) {
+ std::istringstream in("1,\"never closed\r");
+ std::string rec;
+ long long n = 0;
+ ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n));
+ EXPECT_EQ(rec, "1,\"never closed\r");
+ EXPECT_EQ(n, 1);
+ EXPECT_FALSE(tsfile_cli::read_record(in, true, rec, n));
}
TEST(InputFormatTest, ReadRecordTsvIgnoresQuotes) {
// With csv_quotes false a quote is just data; no line joining happens.
- std::istringstream in("1\t\"open\n2\tclosed\n");
+ std::istringstream in("1\t\"open\r\n2\tclosed\r\n");
std::string rec;
long long n = 0;
ASSERT_TRUE(tsfile_cli::read_record(in, false, rec, n));
EXPECT_EQ(rec, "1\t\"open");
EXPECT_EQ(n, 1);
+ ASSERT_TRUE(tsfile_cli::read_record(in, false, rec, n));
+ EXPECT_EQ(rec, "2\tclosed");
+ EXPECT_EQ(n, 1);
}
TEST(InputFormatTest, ReadRecordUnterminatedQuoteAtEof) {
diff --git a/cpp/tools/commands/cmd_write.cc b/cpp/tools/commands/cmd_write.cc
index 403b57fa6..5c7a804cc 100644
--- a/cpp/tools/commands/cmd_write.cc
+++ b/cpp/tools/commands/cmd_write.cc
@@ -562,7 +562,7 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/,
return kExitFile;
}
has_input_stat = true;
- fin.open(args.file.c_str());
+ fin.open(args.file.c_str(), std::ios::binary);
if (!fin.is_open()) {
err << "Error: cannot open input: " << args.file << "\n";
return kExitFile;
diff --git a/cpp/tools/format/input_format.cc b/cpp/tools/format/input_format.cc
index d89368df3..53aee599c 100644
--- a/cpp/tools/format/input_format.cc
+++ b/cpp/tools/format/input_format.cc
@@ -44,23 +44,26 @@ bool read_record(std::istream& in, bool csv_quotes, std::string& record,
std::string physical;
bool open_quote = false;
while (std::getline(in, physical)) {
- strip_cr(physical);
++lines_consumed;
- if (!record.empty()) {
- record.push_back('\n'); // restore the newline inside the field
- }
- record += physical;
if (csv_quotes) {
for (char c : physical) {
if (c == '"') {
open_quote = !open_quote;
}
}
- if (open_quote) {
- continue; // quote still open: the field spans the next line
- }
}
- return true;
+ if (!open_quote) {
+ // Strip CR from record boundaries, but keep it as field data
+ // when the quoted field continues onto the next physical line.
+ strip_cr(physical);
+ }
+ if (!record.empty()) {
+ record.push_back('\n'); // restore the newline inside the field
+ }
+ record += physical;
+ if (!open_quote) {
+ return true;
+ }
}
return lines_consumed > 0; // trailing record with an unterminated quote
}
@@ -80,8 +83,7 @@ std::vector<std::string> split_line(const std::string& line, char delim,
}
bool normalize_write_columns(const std::vector<WriteColumnSpec>& specs,
- std::vector<ColumnDef>& out,
- std::string& error) {
+ std::vector<ColumnDef>& out, std::string& error) {
out.clear();
if (specs.empty()) {
error = "empty write column declarations";
diff --git a/cpp/tools/format/input_format.h b/cpp/tools/format/input_format.h
index 5f76d3fb9..45afd9811 100644
--- a/cpp/tools/format/input_format.h
+++ b/cpp/tools/format/input_format.h
@@ -38,8 +38,7 @@ struct ColumnDef {
bool parse_datatype_name(const std::string& s, common::TSDataType& out);
bool normalize_write_columns(const std::vector<WriteColumnSpec>& specs,
- std::vector<ColumnDef>& out,
- std::string& error);
+ std::vector<ColumnDef>& out, std::string& error);
std::vector<std::string> split_line(const std::string& line, char delim,
bool csv_quotes);
bool parse_bool_cell(const std::string& s, bool& out);
@@ -49,9 +48,12 @@ bool validate_identifier(const std::string& name, std::string& error);
// Read one logical record from `in`. When `csv_quotes` is true a field may span
// multiple physical lines if it opens a double-quote that is not yet closed, so
-// continuation lines are appended (newline preserved) until the quote closes.
-// `lines_consumed` returns how many physical lines were read, so callers can
-// keep accurate line numbers. Returns false at end of input with no record.
+// continuation lines are appended (LF or CRLF preserved) until the quote
+// closes. CRLF outside quotes is a record boundary, not field data. File
+// streams should be opened in binary mode to prevent platform newline
+// translation. `lines_consumed` returns how many physical lines were read, so
+// callers can keep accurate line numbers. Returns false at end of input with no
+// record.
bool read_record(std::istream& in, bool csv_quotes, std::string& record,
long long& lines_consumed);
diff --git a/cpp/tools/tools_main.cc b/cpp/tools/tools_main.cc
index 785c77cce..6cdb60ac3 100644
--- a/cpp/tools/tools_main.cc
+++ b/cpp/tools/tools_main.cc
@@ -19,7 +19,12 @@
#include <exception>
#include <iostream>
-#ifndef _WIN32
+#ifdef _WIN32
+#include <fcntl.h>
+#include <io.h>
+
+#include <cstdio>
+#else
#include <csignal>
#endif
#include <string>
@@ -29,7 +34,19 @@
#include "cli/run_cli.h"
int main(int argc, char** argv) {
-#ifndef _WIN32
+#ifdef _WIN32
+ // Preserve CSV field bytes in pipes and redirected files. CRT text mode
+ // otherwise converts CRLF on input and LF on output, including inside
+ // quotes.
+ const auto binary_if_redirected = [](FILE* stream) {
+ const int fd = _fileno(stream);
+ if (fd >= 0 && _isatty(fd) == 0) {
+ _setmode(fd, _O_BINARY);
+ }
+ };
+ binary_if_redirected(stdin);
+ binary_if_redirected(stdout);
+#else
std::signal(SIGPIPE, SIG_IGN);
#endif
std::vector<std::string> args(argv + 1, argv + argc);