You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2021/04/08 20:47:02 UTC
[impala] 03/03: IMPALA-9997/IMPALA-9998: Upgrade compression
libraries to latest versions
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit d7cc510c95c4850190ca02ae1397aef95cde3d98
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Wed Nov 25 12:36:13 2020 -0800
IMPALA-9997/IMPALA-9998: Upgrade compression libraries to latest versions
This updates several compression libraries to their latest versions:
- Bzip2 1.0.8
- LZ4 1.9.3
- Snappy 1.1.8
- Zlib 1.2.11
- ZStd 1.4.9
Several of these claim minor performance improvements.
Testing:
- Ran release exhaustive job and debug core job
- Ran TPC-H scale 42 with Parquet/Snappy and Parquet/ZSTD.
(ZSTD tests ran with default compression level.)
Parquet/Snappy was unchanged. Parquet/ZSTD improved:
+----------+------------------------+---------+------------+------------+----------------+
| Workload | File Format | Avg (s) | Delta(Avg) | GeoMean(s) | Delta(GeoMean) |
+----------+------------------------+---------+------------+------------+----------------+
| TPCH(42) | parquet / zstd / block | 8.50 | -2.10% | 5.46 | -2.63% |
+----------+------------------------+---------+------------+------------+----------------+
Change-Id: I858f82f773023bd0aea14543f18bd74071758468
Reviewed-on: http://gerrit.cloudera.org:8080/17254
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/util/compress.cc | 1 -
be/src/util/decompress.cc | 1 -
bin/impala-config.sh | 12 ++++++------
3 files changed, 6 insertions(+), 8 deletions(-)
diff --git a/be/src/util/compress.cc b/be/src/util/compress.cc
index 9166d83..2e5128e 100644
--- a/be/src/util/compress.cc
+++ b/be/src/util/compress.cc
@@ -25,7 +25,6 @@
#include <bzlib.h>
#include <zlib.h>
#include <boost/crc.hpp>
-#undef DISALLOW_COPY_AND_ASSIGN // Snappy redefines this.
#include <lz4.h>
#include <snappy.h>
#include <zconf.h>
diff --git a/be/src/util/decompress.cc b/be/src/util/decompress.cc
index 8430328..fbfcba5 100644
--- a/be/src/util/decompress.cc
+++ b/be/src/util/decompress.cc
@@ -24,7 +24,6 @@
// Codec libraries
#include <zlib.h>
#include <bzlib.h>
-#undef DISALLOW_COPY_AND_ASSIGN // Snappy redefines this.
#include <lz4.h>
#include <snappy.h>
#include <zstd.h>
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 2e3166e..d807860 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -68,7 +68,7 @@ fi
# moving to a different build of the toolchain, e.g. when a version is bumped or a
# compile option is changed. The build id can be found in the output of the toolchain
# build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=75-4ad03513e3
+export IMPALA_TOOLCHAIN_BUILD_ID=10-425e7fd862
# Versions of toolchain dependencies.
# -----------------------------------
export IMPALA_AVRO_VERSION=1.7.4-p5
@@ -79,7 +79,7 @@ export IMPALA_BOOST_VERSION=1.61.0-p2
unset IMPALA_BOOST_URL
export IMPALA_BREAKPAD_VERSION=97a98836768f8f0154f8f86e5e14c2bb7e74132e-p2
unset IMPALA_BREAKPAD_URL
-export IMPALA_BZIP2_VERSION=1.0.6-p2
+export IMPALA_BZIP2_VERSION=1.0.8-p2
unset IMPALA_BZIP2_URL
export IMPALA_CCTZ_VERSION=2.2
unset IMPALA_CCTZ_URL
@@ -124,9 +124,9 @@ export IMPALA_LLVM_UBSAN_BASE_VERSION=5.0.1
# Don't use the LLVM debug build because the binaries are too large to distribute.
export IMPALA_LLVM_DEBUG_VERSION=5.0.1-asserts-p3
unset IMPALA_LLVM_DEBUG_URL
-export IMPALA_LZ4_VERSION=1.7.5
+export IMPALA_LZ4_VERSION=1.9.3
unset IMPALA_LZ4_URL
-export IMPALA_ZSTD_VERSION=1.4.0
+export IMPALA_ZSTD_VERSION=1.4.9
unset IMPALA_ZSTD_URL
export IMPALA_OPENLDAP_VERSION=2.4.47
unset IMPALA_OPENLDAP_URL
@@ -144,7 +144,7 @@ export IMPALA_RAPIDJSON_VERSION=1.1.0
unset IMPALA_RAPIDJSON_URL
export IMPALA_RE2_VERSION=20190301
unset IMPALA_RE2_URL
-export IMPALA_SNAPPY_VERSION=1.1.4
+export IMPALA_SNAPPY_VERSION=1.1.8
unset IMPALA_SNAPPY_URL
export IMPALA_SQUEASEL_VERSION=3.3
unset IMPALA_SQUEASEL_URL
@@ -157,7 +157,7 @@ export IMPALA_THRIFT_VERSION=0.9.3-p8
unset IMPALA_THRIFT_URL
export IMPALA_THRIFT11_VERSION=0.11.0-p4
unset IMPALA_THRIFT11_URL
-export IMPALA_ZLIB_VERSION=1.2.8
+export IMPALA_ZLIB_VERSION=1.2.11
unset IMPALA_ZLIB_URL
if [[ $OSTYPE == "darwin"* ]]; then