You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2024/10/21 08:48:16 UTC
(doris) branch branch-2.1 updated: [cherry-pick](branch-2.1) support reading brotli compressed parquet file (#42162)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new a32ad0b1f79 [cherry-pick](branch-2.1) support reading brotli compressed parquet file (#42162)
a32ad0b1f79 is described below
commit a32ad0b1f793d7d795aa26ce937e6d773e2700da
Author: Socrates <su...@icloud.com>
AuthorDate: Mon Oct 21 16:48:09 2024 +0800
[cherry-pick](branch-2.1) support reading brotli compressed parquet file (#42162)
pick pr: https://github.com/apache/doris/pull/41875
---
be/src/util/block_compression.cpp | 37 +++++++++++++++++----
.../hdfs_tvf/test_parquet.brotli.parquet | Bin 0 -> 291443 bytes
.../tvf/test_hdfs_parquet_group0.out | Bin 23993 -> 24011 bytes
.../data/external_table_p0/tvf/test_hdfs_tvf.out | 29 ++++++++++++++++
.../tvf/test_hdfs_parquet_group0.groovy | 7 ++--
.../external_table_p0/tvf/test_hdfs_tvf.groovy | 8 +++++
6 files changed, 69 insertions(+), 12 deletions(-)
diff --git a/be/src/util/block_compression.cpp b/be/src/util/block_compression.cpp
index 8959e9e6b2c..8445ea177ab 100644
--- a/be/src/util/block_compression.cpp
+++ b/be/src/util/block_compression.cpp
@@ -28,24 +28,23 @@
defined(__i386) || defined(_M_IX86)
#include <libdeflate.h>
#endif
+#include <brotli/decode.h>
#include <glog/log_severity.h>
#include <glog/logging.h>
-#include <limits.h>
#include <lz4/lz4.h>
#include <lz4/lz4frame.h>
#include <lz4/lz4hc.h>
#include <snappy/snappy-sinksource.h>
#include <snappy/snappy.h>
-#include <stdint.h>
#include <zconf.h>
#include <zlib.h>
#include <zstd.h>
#include <zstd_errors.h>
#include <algorithm>
+#include <cstdint>
#include <limits>
#include <mutex>
-#include <new>
#include <ostream>
#include "common/config.h"
@@ -53,9 +52,7 @@
#include "exec/decompressor.h"
#include "gutil/endian.h"
#include "gutil/strings/substitute.h"
-#include "orc/OrcFile.hh"
#include "runtime/thread_context.h"
-#include "util/bit_util.h"
#include "util/defer_op.h"
#include "util/faststring.h"
@@ -74,8 +71,6 @@ uint64_t lzoDecompress(const char* inputAddress, const char* inputLimit, char* o
namespace doris {
-using strings::Substitute;
-
// exception safe
Status BlockCompressionCodec::compress(const std::vector<Slice>& inputs, size_t uncompressed_size,
faststring* output) {
@@ -1492,6 +1487,31 @@ public:
}
};
+class BrotliBlockCompression final : public BlockCompressionCodec {
+public:
+ static BrotliBlockCompression* instance() {
+ static BrotliBlockCompression s_instance;
+ return &s_instance;
+ }
+
+ Status compress(const Slice& input, faststring* output) override {
+ return Status::InvalidArgument("not impl brotli compress.");
+ }
+
+ size_t max_compressed_len(size_t len) override { return 0; };
+
+ Status decompress(const Slice& input, Slice* output) override {
+ // The size of output buffer is always equal to the umcompressed length.
+ BrotliDecoderResult result = BrotliDecoderDecompress(
+ input.get_size(), reinterpret_cast<const uint8_t*>(input.get_data()), &output->size,
+ reinterpret_cast<uint8_t*>(output->data));
+ if (result != BROTLI_DECODER_RESULT_SUCCESS) {
+ return Status::InternalError("Brotli decompression failed, result={}", result);
+ }
+ return Status::OK();
+ }
+};
+
Status get_block_compression_codec(segment_v2::CompressionTypePB type,
BlockCompressionCodec** codec) {
switch (type) {
@@ -1582,6 +1602,9 @@ Status get_block_compression_codec(tparquet::CompressionCodec::type parquet_code
case tparquet::CompressionCodec::LZO:
*codec = LzoBlockCompression::instance();
break;
+ case tparquet::CompressionCodec::BROTLI:
+ *codec = BrotliBlockCompression::instance();
+ break;
default:
return Status::InternalError("unknown compression type({})", parquet_codec);
}
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/hdfs_tvf/test_parquet.brotli.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/hdfs_tvf/test_parquet.brotli.parquet
new file mode 100644
index 00000000000..be60868a398
Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/hdfs_tvf/test_parquet.brotli.parquet differ
diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group0.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group0.out
index 6b58c1478b1..7ed43af1f35 100644
Binary files a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group0.out and b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group0.out differ
diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_tvf.out b/regression-test/data/external_table_p0/tvf/test_hdfs_tvf.out
index 68e310c5a4f..d9d8ba4177f 100644
--- a/regression-test/data/external_table_p0/tvf/test_hdfs_tvf.out
+++ b/regression-test/data/external_table_p0/tvf/test_hdfs_tvf.out
@@ -221,6 +221,35 @@
19 Supplier#000000019 edZT3es,nBFD8lBXTGeTl 24 34-278-310-2731 6150.38 refully final foxes across the dogged theodolites sleep slyly abou
20 Supplier#000000020 iybAE,RmTymrZVYaFZva2SH,j 3 13-715-945-6730 530.82 n, ironic ideas would nag blithely about the slyly regular accounts. silent, expr
+-- !parquet_brotli --
+1 Supplier#000000001 N kD4on9OM Ipw3,gf0JBoQDd7tgrzrddZ 17 27-918-335-1736 5755.94 each slyly above the careful
+2 Supplier#000000002 89eJ5ksX3ImxJQBvxObC, 5 15-679-861-2259 4032.68 slyly bold instructions. idle dependen
+3 Supplier#000000003 q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3 1 11-383-516-1199 4192.40 blithely silent requests after the express dependencies are sl
+4 Supplier#000000004 Bk7ah4CK8SYQTepEmvMkkgMwg 15 25-843-787-7479 4641.08 riously even requests above the exp
+5 Supplier#000000005 Gcdm2rJRzl5qlTVzc 11 21-151-690-3663 -283.84 . slyly regular pinto bea
+6 Supplier#000000006 tQxuVm7s7CnK 14 24-696-997-4969 1365.79 final accounts. regular dolphins use against the furiously ironic decoys.
+7 Supplier#000000007 s,4TicNGB4uO6PaSqNBUq 23 33-990-965-2201 6820.35 s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit
+8 Supplier#000000008 9Sq4bBH2FQEmaFOocY45sRTxo6yuoG 17 27-498-742-3860 7627.85 al pinto beans. asymptotes haggl
+9 Supplier#000000009 1KhUgZegwM3ua7dsYmekYBsK 10 20-403-398-8662 5302.37 s. unusual, even requests along the furiously regular pac
+10 Supplier#000000010 Saygah3gYWMp72i PY 24 34-852-489-8585 3891.91 ing waters. regular requests ar
+11 Supplier#000000011 JfwTs,LZrV, M,9C 18 28-613-996-1505 3393.08 y ironic packages. slyly ironic accounts affix furiously; ironically unusual excuses across the flu
+12 Supplier#000000012 aLIW q0HYd 8 18-179-925-7181 1432.69 al packages nag alongside of the bold instructions. express, daring accounts
+13 Supplier#000000013 HK71HQyWoqRWOX8GI FpgAifW,2PoH 3 13-727-620-7813 9107.22 requests engage regularly instructions. furiously special requests ar
+14 Supplier#000000014 EXsnO5pTNj4iZRm 15 25-656-247-5058 9189.82 l accounts boost. fluffily bold warhorses wake
+15 Supplier#000000015 olXVbNBfVzRqgokr1T,Ie 8 18-453-357-6394 308.56 across the furiously regular platelets wake even deposits. quickly express she
+16 Supplier#000000016 YjP5C55zHDXL7LalK27zfQnwejdpin4AMpvh 22 32-822-502-4215 2972.26 ously express ideas haggle quickly dugouts? fu
+17 Supplier#000000017 c2d,ESHRSkK3WYnxpgw6aOqN0q 19 29-601-884-9219 1687.81 eep against the furiously bold ideas. fluffily bold packa
+18 Supplier#000000018 PGGVE5PWAMwKDZw 16 26-729-551-1115 7040.82 accounts snooze slyly furiously bold
+19 Supplier#000000019 edZT3es,nBFD8lBXTGeTl 24 34-278-310-2731 6150.38 refully final foxes across the dogged theodolites sleep slyly abou
+20 Supplier#000000020 iybAE,RmTymrZVYaFZva2SH,j 3 13-715-945-6730 530.82 n, ironic ideas would nag blithely about the slyly regular accounts. silent, expr
+
+-- !parquet_decimal256 --
+1 99999999999999999999999999999999999999.99999999999999999999999999999999999999
+2 -99999999999999999999999999999999999999.99999999999999999999999999999999999999
+3 1E-38
+4 -1E-38
+5 0E-38
+
-- !orc --
1 goldenrod lavender spring chocolate lace Manufacturer#1 Brand#13 PROMO BURNISHED COPPER 7 JUMBO PKG 901.00 ly. slyly ironi
2 blush thistle blue yellow saddle Manufacturer#1 Brand#13 LARGE BRUSHED BRASS 1 LG CASE 902.00 lar accounts amo
diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy
index 2af8eef6cb5..2bf95551a38 100644
--- a/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_parquet_group0.groovy
@@ -104,13 +104,10 @@ suite("test_hdfs_parquet_group0","external,hive,tvf,external_docker") {
uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group0/large_string_map.brotli.parquet"
- test {
- sql """ select * from HDFS(
+ order_qt_test_11 """ select count(arr) from HDFS(
"uri" = "${uri}",
"hadoop.username" = "${hdfsUserName}",
- "format" = "parquet") limit 10; """
- exception "unknown compression type(4)"
- }
+ "format" = "parquet"); """
uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_parquet/group0/non_hadoop_lz4_compressed.parquet"
diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy
index abdfd871a1f..be27e213f6b 100644
--- a/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy
@@ -107,6 +107,14 @@ suite("test_hdfs_tvf","external,hive,tvf,external_docker") {
"uri" = "${uri}",
"hadoop.username" = "${hdfsUserName}",
"format" = "${format}") order by s_suppkey limit 20; """
+
+ // test parquet brotli
+ uri = "${defaultFS}" + "/user/doris/preinstalled_data/hdfs_tvf/test_parquet.brotli.parquet"
+ format = "parquet"
+ qt_parquet_brotli """ select * from HDFS(
+ "uri" = "${uri}",
+ "hadoop.username" = "${hdfsUserName}",
+ "format" = "${format}") order by s_suppkey limit 20; """
// test orc
uri = "${defaultFS}" + "/user/doris/preinstalled_data/hdfs_tvf/test_orc.snappy.orc"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org