You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zh...@apache.org on 2019/12/27 00:06:29 UTC
[incubator-doris] branch master updated: Support decompressing csv
file with deflate format in hdfs broker load (#2583)
This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 043a952 Support decompressing csv file with deflate format in hdfs broker load (#2583)
043a952 is described below
commit 043a9528f7172f1ecc36a48c590f12a3c64c0c40
Author: caiconghui <55...@users.noreply.github.com>
AuthorDate: Fri Dec 27 08:06:22 2019 +0800
Support decompressing csv file with deflate format in hdfs broker load (#2583)
---
be/src/exec/broker_scanner.cpp | 4 ++++
fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java | 2 ++
gensrc/thrift/PlanNodes.thrift | 3 ++-
3 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/be/src/exec/broker_scanner.cpp b/be/src/exec/broker_scanner.cpp
index 7db541a..1155db9 100644
--- a/be/src/exec/broker_scanner.cpp
+++ b/be/src/exec/broker_scanner.cpp
@@ -194,6 +194,9 @@ Status BrokerScanner::create_decompressor(TFileFormatType::type type) {
case TFileFormatType::FORMAT_CSV_LZOP:
compress_type = CompressType::LZOP;
break;
+ case TFileFormatType::FORMAT_CSV_DEFLATE:
+ compress_type = CompressType::DEFLATE;
+ break;
default: {
std::stringstream ss;
ss << "Unknown format type, type=" << type;
@@ -242,6 +245,7 @@ Status BrokerScanner::open_line_reader() {
case TFileFormatType::FORMAT_CSV_BZ2:
case TFileFormatType::FORMAT_CSV_LZ4FRAME:
case TFileFormatType::FORMAT_CSV_LZOP:
+ case TFileFormatType::FORMAT_CSV_DEFLATE:
_cur_line_reader = new PlainTextLineReader(
_profile,
_cur_file_reader, _cur_decompressor,
diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java
index 1e9f5a3..680f53a 100644
--- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java
+++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java
@@ -415,6 +415,8 @@ public class BrokerScanNode extends LoadScanNode {
return TFileFormatType.FORMAT_CSV_LZ4FRAME;
} else if (lowerCasePath.endsWith(".lzo")) {
return TFileFormatType.FORMAT_CSV_LZOP;
+ } else if (lowerCasePath.endsWith(".deflate")) {
+ return TFileFormatType.FORMAT_CSV_DEFLATE;
} else {
return TFileFormatType.FORMAT_CSV_PLAIN;
}
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 5944dda..8d29c4d 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -98,7 +98,8 @@ enum TFileFormatType {
FORMAT_CSV_BZ2,
FORMAT_CSV_LZ4FRAME,
FORMAT_CSV_LZOP,
- FORMAT_PARQUET
+ FORMAT_PARQUET,
+ FORMAT_CSV_DEFLATE
}
// One broker range information.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org