You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ga...@apache.org on 2022/10/23 05:02:57 UTC
[doris] branch master updated: [fix](export)(outfile) fix bug that export may fail when writing SUCCESS file (#13574)
This is an automated email from the ASF dual-hosted git repository.
gabriellee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4b5a2c1a65 [fix](export)(outfile) fix bug that export may fail when writing SUCCESS file (#13574)
4b5a2c1a65 is described below
commit 4b5a2c1a65dbc718d45e70b2f0cae8205b4cef24
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Sun Oct 23 13:02:49 2022 +0800
[fix](export)(outfile) fix bug that export may fail when writing SUCCESS file (#13574)
---
be/src/vec/runtime/vfile_result_writer.cpp | 9 +++++++--
be/src/vec/runtime/vfile_result_writer.h | 2 +-
regression-test/suites/export_p0/test_outfile.groovy | 7 ++++---
3 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/be/src/vec/runtime/vfile_result_writer.cpp b/be/src/vec/runtime/vfile_result_writer.cpp
index 973f79de27..990a48d65d 100644
--- a/be/src/vec/runtime/vfile_result_writer.cpp
+++ b/be/src/vec/runtime/vfile_result_writer.cpp
@@ -80,7 +80,8 @@ Status VFileResultWriter::_create_success_file() {
std::string file_name;
RETURN_IF_ERROR(_get_success_file_name(&file_name));
RETURN_IF_ERROR(_create_file_writer(file_name));
- return _close_file_writer(true);
+ // set only close to true to avoid dead loop
+ return _close_file_writer(true, true);
}
Status VFileResultWriter::_get_success_file_name(std::string* file_name) {
@@ -422,7 +423,7 @@ Status VFileResultWriter::_create_new_file_if_exceed_size() {
return Status::OK();
}
-Status VFileResultWriter::_close_file_writer(bool done) {
+Status VFileResultWriter::_close_file_writer(bool done, bool only_close) {
if (_vfile_writer) {
_vfile_writer->close();
COUNTER_UPDATE(_written_data_bytes, _current_written_bytes);
@@ -431,6 +432,10 @@ Status VFileResultWriter::_close_file_writer(bool done) {
_file_writer_impl->close();
}
+ if (only_close) {
+ return Status::OK();
+ }
+
if (!done) {
// not finished, create new file writer for next file
RETURN_IF_ERROR(_create_next_file_writer());
diff --git a/be/src/vec/runtime/vfile_result_writer.h b/be/src/vec/runtime/vfile_result_writer.h
index 277fe07789..d16946491a 100644
--- a/be/src/vec/runtime/vfile_result_writer.h
+++ b/be/src/vec/runtime/vfile_result_writer.h
@@ -70,7 +70,7 @@ private:
std::string _file_format_to_name();
// close file writer, and if !done, it will create new writer for next file.
// if only_close is true, this method will just close the file writer and return.
- Status _close_file_writer(bool done);
+ Status _close_file_writer(bool done, bool only_close = false);
// create a new file if current file size exceed limit
Status _create_new_file_if_exceed_size();
// send the final statistic result
diff --git a/regression-test/suites/export_p0/test_outfile.groovy b/regression-test/suites/export_p0/test_outfile.groovy
index 0dfd86d036..9d2b3c561b 100644
--- a/regression-test/suites/export_p0/test_outfile.groovy
+++ b/regression-test/suites/export_p0/test_outfile.groovy
@@ -163,12 +163,13 @@ suite("test_outfile") {
sql "set return_object_data_as_binary = false"
sql """
- SELECT * FROM ${tableName} t ORDER BY k1, v2 INTO OUTFILE "file://${outFilePath}/";
+ SELECT * FROM ${tableName} t ORDER BY k1, v2 INTO OUTFILE "file://${outFilePath}/" properties("success_file_name" = "SUCCESS")
"""
File[] files = path.listFiles()
- assert files.length == 1
- List<String> outLines = Files.readAllLines(Paths.get(files[0].getAbsolutePath()), StandardCharsets.UTF_8);
+ assert files.length == 2 // one is outfile, the other is SUCCESS file
+ File dataFile = files[0].getName().contains("SUCCESS") ? files[1] : files[0];
+ List<String> outLines = Files.readAllLines(Paths.get(dataFile.getAbsolutePath()), StandardCharsets.UTF_8);
assertEquals(2, outLines.size())
String[] outLine1 = outLines.get(0).split("\t")
assertEquals(3, outLine1.size())
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org