You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/03/28 04:26:30 UTC
[7/7] impala git commit: IMPALA-5980: Upgrade to LLVM 5.0.1
IMPALA-5980: Upgrade to LLVM 5.0.1
Highlighting a few changes in LLVM:
- Minor changes to some function signatures
- Minor changes to error handling
- Split Bitcode/ReaderWriter.h - https://reviews.llvm.org/D26502
- Introduced an optional new GVN optimization pass.
Needed to fix a bunch of new clang-tidy warnings.
Testing:
Ran core and ASAN tests successfully.
Performance:
Ran single node TPC-H and targeted perf with scale factor 60. Both
improved on average. Identified regression in
"primitive_filter_in_predicate" which will be addressed by IMPALA-6621.
+-------------------+-----------------------+---------+------------+------------+----------------+
| Workload | File Format | Avg (s) | Delta(Avg) | GeoMean(s) | Delta(GeoMean) |
+-------------------+-----------------------+---------+------------+------------+----------------+
| TARGETED-PERF(60) | parquet / none / none | 22.29 | -0.12% | 3.90 | +3.16% |
| TPCH(60) | parquet / none / none | 15.97 | -3.64% | 10.14 | -4.92% |
+-------------------+-----------------------+---------+------------+------------+----------------+
+-------------------+--------------------------------------------------------+-----------------------+--------+-------------+------------+------------+----------------+-------------+-------+
| Workload | Query | File Format | Avg(s) | Base Avg(s) | Delta(Avg) | StdDev(%) | Base StdDev(%) | Num Clients | Iters |
+-------------------+--------------------------------------------------------+-----------------------+--------+-------------+------------+------------+----------------+-------------+-------+
| TARGETED-PERF(60) | PERF_LIMIT-Q1 | parquet / none / none | 0.01 | 0.00 | R +156.43% | * 25.80% * | * 17.14% * | 1 | 5 |
| TARGETED-PERF(60) | primitive_filter_in_predicate | parquet / none / none | 3.39 | 1.92 | R +76.33% | 3.23% | 4.37% | 1 | 5 |
| TARGETED-PERF(60) | primitive_filter_string_non_selective | parquet / none / none | 1.25 | 1.11 | +12.46% | 3.41% | 5.36% | 1 | 5 |
| TARGETED-PERF(60) | primitive_filter_decimal_selective | parquet / none / none | 1.40 | 1.25 | +12.25% | 3.57% | 3.44% | 1 | 5 |
| TARGETED-PERF(60) | primitive_filter_string_like | parquet / none / none | 16.87 | 15.65 | +7.78% | 5.05% | 0.37% | 1 | 5 |
| TARGETED-PERF(60) | primitive_min_max_runtime_filter | parquet / none / none | 1.79 | 1.71 | +4.77% | 0.71% | 1.73% | 1 | 5 |
| TARGETED-PERF(60) | primitive_broadcast_join_2 | parquet / none / none | 0.60 | 0.58 | +3.64% | 3.19% | 3.81% | 1 | 5 |
| TARGETED-PERF(60) | primitive_filter_string_selective | parquet / none / none | 0.95 | 0.93 | +2.91% | 5.23% | 5.85% | 1 | 5 |
| TARGETED-PERF(60) | primitive_broadcast_join_3 | parquet / none / none | 4.33 | 4.21 | +2.83% | 5.46% | 3.25% | 1 | 5 |
| TARGETED-PERF(60) | primitive_groupby_bigint_lowndv | parquet / none / none | 4.59 | 4.47 | +2.82% | 3.73% | 1.14% | 1 | 5 |
| TARGETED-PERF(60) | primitive_conjunct_ordering_3 | parquet / none / none | 0.20 | 0.19 | +2.65% | 4.76% | 2.24% | 1 | 5 |
| TARGETED-PERF(60) | PERF_AGG-Q1 | parquet / none / none | 2.49 | 2.43 | +2.31% | 1.06% | 1.93% | 1 | 5 |
| TARGETED-PERF(60) | PERF_AGG-Q6 | parquet / none / none | 2.04 | 2.00 | +2.09% | 3.51% | 2.80% | 1 | 5 |
| TPCH(60) | TPCH-Q3 | parquet / none / none | 12.37 | 12.17 | +1.62% | 0.80% | 2.45% | 1 | 5 |
| TARGETED-PERF(60) | PERF_STRING-Q5 | parquet / none / none | 4.52 | 4.45 | +1.54% | 1.23% | 1.08% | 1 | 5 |
| TPCH(60) | TPCH-Q6 | parquet / none / none | 2.95 | 2.91 | +1.33% | 1.92% | 1.67% | 1 | 5 |
| TARGETED-PERF(60) | PERF_STRING-Q4 | parquet / none / none | 3.71 | 3.66 | +1.26% | 0.34% | 0.53% | 1 | 5 |
| TPCH(60) | TPCH-Q1 | parquet / none / none | 18.69 | 18.47 | +1.19% | 0.75% | 0.31% | 1 | 5 |
| TARGETED-PERF(60) | PERF_STRING-Q7 | parquet / none / none | 8.15 | 8.07 | +0.99% | 3.92% | 1.58% | 1 | 5 |
| TARGETED-PERF(60) | primitive_groupby_decimal_highndv | parquet / none / none | 31.31 | 31.01 | +0.97% | 1.74% | 1.14% | 1 | 5 |
| TPCH(60) | TPCH-Q5 | parquet / none / none | 7.59 | 7.53 | +0.78% | 0.38% | 0.99% | 1 | 5 |
| TARGETED-PERF(60) | PERF_AGG-Q4 | parquet / none / none | 21.25 | 21.09 | +0.76% | 0.76% | 0.75% | 1 | 5 |
| TARGETED-PERF(60) | primitive_conjunct_ordering_4 | parquet / none / none | 0.24 | 0.24 | +0.75% | 3.14% | 4.76% | 1 | 5 |
| TPCH(60) | TPCH-Q19 | parquet / none / none | 7.88 | 7.82 | +0.74% | 2.39% | 2.64% | 1 | 5 |
| TARGETED-PERF(60) | primitive_orderby_bigint | parquet / none / none | 5.10 | 5.07 | +0.61% | 0.74% | 0.54% | 1 | 5 |
| TARGETED-PERF(60) | PERF_STRING-Q3 | parquet / none / none | 3.61 | 3.59 | +0.60% | 1.45% | 0.90% | 1 | 5 |
| TARGETED-PERF(60) | primitive_orderby_all | parquet / none / none | 27.63 | 27.48 | +0.55% | 0.85% | 0.10% | 1 | 5 |
| TPCH(60) | TPCH-Q4 | parquet / none / none | 5.81 | 5.79 | +0.45% | 1.65% | 2.16% | 1 | 5 |
| TPCH(60) | TPCH-Q13 | parquet / none / none | 23.49 | 23.43 | +0.27% | 0.83% | 0.63% | 1 | 5 |
| TPCH(60) | TPCH-Q21 | parquet / none / none | 68.88 | 68.76 | +0.18% | 0.22% | 0.19% | 1 | 5 |
| TARGETED-PERF(60) | primitive_groupby_decimal_lowndv.test | parquet / none / none | 4.38 | 4.37 | +0.09% | 2.45% | 0.45% | 1 | 5 |
| TARGETED-PERF(60) | primitive_conjunct_ordering_5 | parquet / none / none | 10.40 | 10.40 | +0.07% | 0.77% | 0.50% | 1 | 5 |
| TARGETED-PERF(60) | primitive_long_predicate | parquet / none / none | 222.37 | 222.23 | +0.06% | 0.25% | 0.25% | 1 | 5 |
| TPCH(60) | TPCH-Q8 | parquet / none / none | 10.65 | 10.65 | +0.03% | 0.55% | 1.40% | 1 | 5 |
| TARGETED-PERF(60) | primitive_shuffle_join_one_to_many_string_with_groupby | parquet / none / none | 261.84 | 261.87 | -0.01% | 0.91% | 0.74% | 1 | 5 |
| TARGETED-PERF(60) | PERF_AGG-Q3 | parquet / none / none | 9.44 | 9.45 | -0.02% | 0.92% | 1.33% | 1 | 5 |
| TPCH(60) | TPCH-Q16 | parquet / none / none | 5.21 | 5.21 | -0.02% | 1.46% | 1.64% | 1 | 5 |
| TARGETED-PERF(60) | primitive_top-n_all | parquet / none / none | 34.58 | 34.62 | -0.11% | 0.22% | 0.19% | 1 | 5 |
| TARGETED-PERF(60) | primitive_topn_bigint | parquet / none / none | 4.24 | 4.25 | -0.13% | 6.66% | 2.03% | 1 | 5 |
| TARGETED-PERF(60) | PERF_STRING-Q2 | parquet / none / none | 3.23 | 3.24 | -0.34% | 2.03% | 0.32% | 1 | 5 |
| TARGETED-PERF(60) | primitive_broadcast_join_1 | parquet / none / none | 0.18 | 0.18 | -0.40% | 6.16% | 2.45% | 1 | 5 |
| TARGETED-PERF(60) | primitive_exchange_broadcast | parquet / none / none | 46.27 | 46.51 | -0.52% | 7.83% | * 15.60% * | 1 | 5 |
| TARGETED-PERF(60) | primitive_groupby_bigint_pk | parquet / none / none | 114.32 | 114.92 | -0.52% | 0.24% | 0.61% | 1 | 5 |
| TPCH(60) | TPCH-Q22 | parquet / none / none | 6.66 | 6.70 | -0.53% | 1.39% | 0.84% | 1 | 5 |
| TPCH(60) | TPCH-Q20 | parquet / none / none | 5.78 | 5.81 | -0.62% | 1.25% | 0.67% | 1 | 5 |
| TPCH(60) | TPCH-Q2 | parquet / none / none | 2.53 | 2.55 | -0.64% | 3.86% | 3.72% | 1 | 5 |
| TARGETED-PERF(60) | PERF_AGG-Q5 | parquet / none / none | 0.58 | 0.58 | -0.75% | 0.99% | 6.89% | 1 | 5 |
| TARGETED-PERF(60) | PERF_AGG-Q7 | parquet / none / none | 2.05 | 2.07 | -0.86% | 2.16% | 4.73% | 1 | 5 |
| TARGETED-PERF(60) | primitive_shuffle_join_union_all_with_groupby | parquet / none / none | 54.86 | 55.34 | -0.87% | 0.25% | 0.66% | 1 | 5 |
| TARGETED-PERF(60) | primitive_conjunct_ordering_2 | parquet / none / none | 7.52 | 7.59 | -0.98% | 1.53% | 1.73% | 1 | 5 |
| TPCH(60) | TPCH-Q9 | parquet / none / none | 36.43 | 36.79 | -1.00% | 1.60% | 7.39% | 1 | 5 |
| TARGETED-PERF(60) | PERF_STRING-Q1 | parquet / none / none | 2.79 | 2.82 | -1.10% | 1.15% | 2.25% | 1 | 5 |
| TPCH(60) | TPCH-Q11 | parquet / none / none | 1.95 | 1.97 | -1.18% | 3.14% | 2.24% | 1 | 5 |
| TARGETED-PERF(60) | PERF_AGG-Q2 | parquet / none / none | 10.98 | 11.11 | -1.24% | 0.77% | 1.45% | 1 | 5 |
| TARGETED-PERF(60) | primitive_small_join_1 | parquet / none / none | 0.22 | 0.22 | -1.34% | * 13.03% * | * 12.31% * | 1 | 5 |
| TPCH(60) | TPCH-Q7 | parquet / none / none | 42.82 | 43.41 | -1.37% | 1.63% | 1.51% | 1 | 5 |
| TARGETED-PERF(60) | primitive_empty_build_join_1 | parquet / none / none | 3.30 | 3.35 | -1.54% | 2.15% | 1.27% | 1 | 5 |
| TARGETED-PERF(60) | PERF_STRING-Q6 | parquet / none / none | 10.34 | 10.54 | -1.81% | 0.24% | 2.02% | 1 | 5 |
| TARGETED-PERF(60) | primitive_groupby_bigint_highndv | parquet / none / none | 32.80 | 33.46 | -1.98% | 1.29% | 0.61% | 1 | 5 |
| TARGETED-PERF(60) | primitive_filter_decimal_non_selective | parquet / none / none | 1.62 | 1.67 | -3.01% | 0.79% | 1.65% | 1 | 5 |
| TARGETED-PERF(60) | primitive_conjunct_ordering_1 | parquet / none / none | 0.13 | 0.14 | -3.36% | 8.66% | * 12.66% * | 1 | 5 |
| TARGETED-PERF(60) | primitive_exchange_shuffle | parquet / none / none | 84.92 | 87.96 | -3.46% | 1.46% | 1.50% | 1 | 5 |
| TPCH(60) | TPCH-Q12 | parquet / none / none | 6.98 | 7.31 | -4.57% | 1.03% | 7.13% | 1 | 5 |
| TPCH(60) | TPCH-Q18 | parquet / none / none | 47.54 | 50.39 | -5.64% | 5.70% | 5.53% | 1 | 5 |
| TARGETED-PERF(60) | primitive_filter_bigint_non_selective | parquet / none / none | 0.88 | 0.96 | -7.81% | 4.27% | 5.97% | 1 | 5 |
| TPCH(60) | TPCH-Q15 | parquet / none / none | 8.14 | 9.15 | -11.09% | 0.63% | * 10.44% * | 1 | 5 |
| TPCH(60) | TPCH-Q10 | parquet / none / none | 12.66 | 14.28 | -11.34% | 4.32% | 1.14% | 1 | 5 |
| TPCH(60) | TPCH-Q17 | parquet / none / none | 10.31 | 12.59 | -18.14% | 0.65% | 3.72% | 1 | 5 |
| TARGETED-PERF(60) | primitive_filter_bigint_selective | parquet / none / none | 0.14 | 0.19 | I -27.60% | * 32.55% * | * 39.78% * | 1 | 5 |
| TPCH(60) | TPCH-Q14 | parquet / none / none | 6.10 | 11.00 | I -44.55% | 4.06% | 3.84% | 1 | 5 |
+-------------------+--------------------------------------------------------+-----------------------+--------+-------------+------------+------------+----------------+-------------+-------+
Change-Id: Ib0a15cb53feab89e7b35a56b67b3b30eb3e62c6b
Reviewed-on: http://gerrit.cloudera.org:8080/9584
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/4a39e7c2
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/4a39e7c2
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/4a39e7c2
Branch: refs/heads/master
Commit: 4a39e7c29f7b910c9ae0d622b355e9362c17aaec
Parents: bc3e4fb
Author: Bikramjeet Vig <bi...@cloudera.com>
Authored: Wed Oct 25 12:12:52 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Mar 28 04:25:27 2018 +0000
----------------------------------------------------------------------
be/CMakeLists.txt | 12 ++++
.../benchmarks/row-batch-serialize-benchmark.cc | 9 ++-
be/src/codegen/codegen-anyval.cc | 14 ++---
be/src/codegen/instruction-counter-test.cc | 4 +-
be/src/codegen/llvm-codegen.cc | 59 ++++++++++++--------
be/src/exprs/aggregate-functions-ir.cc | 32 -----------
be/src/rpc/impala-service-pool.h | 4 +-
be/src/runtime/bufferpool/buffer-pool-test.cc | 6 +-
be/src/runtime/data-stream-sender.cc | 2 -
be/src/runtime/data-stream-sender.h | 3 -
be/src/runtime/krpc-data-stream-sender.cc | 1 -
be/src/runtime/krpc-data-stream-sender.h | 3 -
be/src/runtime/row-batch-serialize-test.cc | 3 +-
be/src/runtime/row-batch.cc | 6 +-
be/src/runtime/row-batch.h | 7 ++-
be/src/runtime/types.cc | 2 +-
be/src/service/fe-support.cc | 28 ++++++----
be/src/thirdparty/squeasel/squeasel.c | 9 ++-
be/src/transport/TSasl.cpp | 26 +++++----
be/src/transport/TSaslClientTransport.cpp | 4 +-
be/src/transport/TSaslTransport.cpp | 2 +-
be/src/udf/udf-test.cc | 3 +-
be/src/udf/udf.h | 4 +-
bin/bootstrap_toolchain.py | 2 +-
bin/impala-config.sh | 8 +--
.../queries/QueryTest/udf-errors.test | 1 +
26 files changed, 131 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 516166d..b6e10b0 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -65,8 +65,20 @@ SET(CXX_COVERAGE_FLAGS "-fprofile-arcs -ftest-coverage")
# when using ccache as clang thinks it is not called from a terminal.
# -Wno-mismatched-tags: ignore harmless class/struct mismatch for forward declarations.
# Disabling to be consistent with gcc, which doesn't have this warning.
+# -Wno-zero-as-null-pointer-constant: We are slowly moving towards the use of nullptr,
+# but till we switch to it completely, we will ignore the warnings due to use of
+# NULL as a null pointer constant.
+# -Wno-c++17-extensions: ignore warnings caused due to the use of [[nodiscard]]
+# attribute which our current compiler does not support but is used in conjunction
+# with WARN_UNUSED_RESULT with our current toolchain to be effective.
+# -Wno-inconsistent-missing-destructor-override: ignore warnings to mark virtual
+# destructors with 'override' which is enforced by clang by not recommended by c++
+# core guidelines (read C.128).
SET(CXX_CLANG_FLAGS "-Qunused-arguments -fcolor-diagnostics -Wno-unused-local-typedef")
SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -Wno-mismatched-tags")
+SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -Wno-zero-as-null-pointer-constant")
+SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -Wno-c++17-extensions")
+SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -Wno-inconsistent-missing-destructor-override")
# For any gcc builds:
# -g: Enable symbols for profiler tools
# -Wno-unused-local-typedefs: Do not warn for local typedefs that are unused.
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/benchmarks/row-batch-serialize-benchmark.cc
----------------------------------------------------------------------
diff --git a/be/src/benchmarks/row-batch-serialize-benchmark.cc b/be/src/benchmarks/row-batch-serialize-benchmark.cc
index a2071ca..b55d04b 100644
--- a/be/src/benchmarks/row-batch-serialize-benchmark.cc
+++ b/be/src/benchmarks/row-batch-serialize-benchmark.cc
@@ -128,8 +128,10 @@ class RowBatchSerializeBaseline {
if (batch->compression_scratch_.size() < compressed_size) {
batch->compression_scratch_.resize(compressed_size);
}
- uint8_t* input = (uint8_t*)output_batch->tuple_data.c_str();
- uint8_t* compressed_output = (uint8_t*)batch->compression_scratch_.c_str();
+ uint8_t* input = const_cast<uint8_t*>(
+ reinterpret_cast<const uint8_t*>(output_batch->tuple_data.c_str()));
+ uint8_t* compressed_output = const_cast<uint8_t*>(
+ reinterpret_cast<const uint8_t*>(batch->compression_scratch_.c_str()));
status =
compressor.ProcessBlock(true, size, input, &compressed_size, &compressed_output);
DCHECK(status.ok()) << status.GetDetail();
@@ -194,7 +196,8 @@ class RowBatchSerializeBaseline {
uint8_t* tuple_data;
if (input_batch.compression_type != THdfsCompression::NONE) {
// Decompress tuple data into data pool
- uint8_t* compressed_data = (uint8_t*)input_batch.tuple_data.c_str();
+ uint8_t* compressed_data = const_cast<uint8_t*>(
+ reinterpret_cast<const uint8_t*>(input_batch.tuple_data.c_str()));
size_t compressed_size = input_batch.tuple_data.size();
Lz4Decompressor decompressor(nullptr, false);
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/codegen/codegen-anyval.cc
----------------------------------------------------------------------
diff --git a/be/src/codegen/codegen-anyval.cc b/be/src/codegen/codegen-anyval.cc
index 6575cfd..3e2eb1e 100644
--- a/be/src/codegen/codegen-anyval.cc
+++ b/be/src/codegen/codegen-anyval.cc
@@ -46,20 +46,20 @@ llvm::Type* CodegenAnyVal::GetLoweredType(LlvmCodeGen* cg, const ColumnType& typ
case TYPE_INT: // i64
return cg->i64_type();
case TYPE_BIGINT: // { i8, i64 }
- return llvm::StructType::get(cg->i8_type(), cg->i64_type(), NULL);
+ return llvm::StructType::get(cg->i8_type(), cg->i64_type());
case TYPE_FLOAT: // i64
return cg->i64_type();
case TYPE_DOUBLE: // { i8, double }
- return llvm::StructType::get(cg->i8_type(), cg->double_type(), NULL);
+ return llvm::StructType::get(cg->i8_type(), cg->double_type());
case TYPE_STRING: // { i64, i8* }
case TYPE_VARCHAR: // { i64, i8* }
case TYPE_FIXED_UDA_INTERMEDIATE: // { i64, i8* }
- return llvm::StructType::get(cg->i64_type(), cg->ptr_type(), NULL);
+ return llvm::StructType::get(cg->i64_type(), cg->ptr_type());
case TYPE_CHAR:
DCHECK(false) << "NYI:" << type.DebugString();
return NULL;
case TYPE_TIMESTAMP: // { i64, i64 }
- return llvm::StructType::get(cg->i64_type(), cg->i64_type(), NULL);
+ return llvm::StructType::get(cg->i64_type(), cg->i64_type());
case TYPE_DECIMAL: // %"struct.impala_udf::DecimalVal" (isn't lowered)
// = { {i8}, [15 x i8], {i128} }
return cg->GetNamedType(LLVM_DECIMALVAL_NAME);
@@ -805,12 +805,12 @@ llvm::Value* CodegenAnyVal::GetNullVal(LlvmCodeGen* codegen, llvm::Type* val_typ
llvm::StructType* anyval_struct_type =
llvm::cast<llvm::StructType>(struct_type->getElementType(0));
llvm::Type* is_null_type = anyval_struct_type->getElementType(0);
- llvm::Value* null_anyval = llvm::ConstantStruct::get(
+ llvm::Constant* null_anyval = llvm::ConstantStruct::get(
anyval_struct_type, llvm::ConstantInt::get(is_null_type, 1));
llvm::Type* type2 = struct_type->getElementType(1);
llvm::Type* type3 = struct_type->getElementType(2);
return llvm::ConstantStruct::get(struct_type, null_anyval,
- llvm::Constant::getNullValue(type2), llvm::Constant::getNullValue(type3), NULL);
+ llvm::Constant::getNullValue(type2), llvm::Constant::getNullValue(type3));
}
// Return the struct { 1, 0 } (the 'is_null' byte, i.e. the first value's first byte,
// is set to 1, the other bytes don't matter)
@@ -819,7 +819,7 @@ llvm::Value* CodegenAnyVal::GetNullVal(LlvmCodeGen* codegen, llvm::Type* val_typ
DCHECK(type1->isIntegerTy()) << LlvmCodeGen::Print(type1);
llvm::Type* type2 = struct_type->getElementType(1);
return llvm::ConstantStruct::get(struct_type, llvm::ConstantInt::get(type1, 1),
- llvm::Constant::getNullValue(type2), NULL);
+ llvm::Constant::getNullValue(type2));
}
// Return the int 1 ('is_null' byte is 1, other bytes don't matter)
DCHECK(val_type->isIntegerTy());
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/codegen/instruction-counter-test.cc
----------------------------------------------------------------------
diff --git a/be/src/codegen/instruction-counter-test.cc b/be/src/codegen/instruction-counter-test.cc
index d719cf3..f404243 100644
--- a/be/src/codegen/instruction-counter-test.cc
+++ b/be/src/codegen/instruction-counter-test.cc
@@ -50,7 +50,7 @@ llvm::Module* CodegenMulAdd(llvm::LLVMContext* context) {
llvm::Module* mod = new llvm::Module("test", *context);
llvm::Constant* c = mod->getOrInsertFunction("mul_add",
llvm::IntegerType::get(*context, 32), llvm::IntegerType::get(*context, 32),
- llvm::IntegerType::get(*context, 32), llvm::IntegerType::get(*context, 32), NULL);
+ llvm::IntegerType::get(*context, 32), llvm::IntegerType::get(*context, 32));
llvm::Function* mul_add = llvm::cast<llvm::Function>(c);
mul_add->setCallingConv(llvm::CallingConv::C);
llvm::Function::arg_iterator args = mul_add->arg_begin();
@@ -116,7 +116,7 @@ llvm::Module* CodegenGcd(llvm::LLVMContext* context) {
llvm::Module* mod = new llvm::Module("gcd", *context);
llvm::Constant* c = mod->getOrInsertFunction("gcd",
llvm::IntegerType::get(*context, 32), llvm::IntegerType::get(*context, 32),
- llvm::IntegerType::get(*context, 32), NULL);
+ llvm::IntegerType::get(*context, 32));
llvm::Function* gcd = llvm::cast<llvm::Function>(c);
llvm::Function::arg_iterator args = gcd->arg_begin();
llvm::Value* x = &*args;
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/codegen/llvm-codegen.cc
----------------------------------------------------------------------
diff --git a/be/src/codegen/llvm-codegen.cc b/be/src/codegen/llvm-codegen.cc
index b3a1c50..5ac19ad 100644
--- a/be/src/codegen/llvm-codegen.cc
+++ b/be/src/codegen/llvm-codegen.cc
@@ -30,7 +30,7 @@
#include <llvm/Analysis/InstructionSimplify.h>
#include <llvm/Analysis/Passes.h>
#include <llvm/Analysis/TargetTransformInfo.h>
-#include <llvm/Bitcode/ReaderWriter.h>
+#include <llvm/Bitcode/BitcodeReader.h>
#include <llvm/ExecutionEngine/ExecutionEngine.h>
#include <llvm/ExecutionEngine/MCJIT.h>
#include <llvm/IR/Constants.h>
@@ -124,7 +124,7 @@ string LlvmCodeGen::target_features_attr_;
CodegenCallGraph LlvmCodeGen::shared_call_graph_;
[[noreturn]] static void LlvmCodegenHandleError(
- void* user_data, const std::string& reason, bool gen_crash_diag) {
+ void* user_data, const string& reason, bool gen_crash_diag) {
LOG(FATAL) << "LLVM hit fatal error: " << reason.c_str();
}
@@ -217,7 +217,7 @@ Status LlvmCodeGen::CreateFromFile(RuntimeState* state, ObjectPool* pool,
unique_ptr<llvm::Module> loaded_module;
Status status = (*codegen)->LoadModuleFromFile(file, &loaded_module);
if (!status.ok()) goto error;
- status = (*codegen)->Init(std::move(loaded_module));
+ status = (*codegen)->Init(move(loaded_module));
if (!status.ok()) goto error;
return Status::OK();
error:
@@ -247,10 +247,10 @@ Status LlvmCodeGen::CreateFromMemory(RuntimeState* state, ObjectPool* pool,
unique_ptr<llvm::MemoryBuffer> module_ir_buf(
llvm::MemoryBuffer::getMemBuffer(module_ir, "", false));
unique_ptr<llvm::Module> loaded_module;
- Status status = (*codegen)->LoadModuleFromMemory(std::move(module_ir_buf),
+ Status status = (*codegen)->LoadModuleFromMemory(move(module_ir_buf),
module_name, &loaded_module);
if (!status.ok()) goto error;
- status = (*codegen)->Init(std::move(loaded_module));
+ status = (*codegen)->Init(move(loaded_module));
if (!status.ok()) goto error;
return Status::OK();
error:
@@ -272,11 +272,11 @@ Status LlvmCodeGen::LoadModuleFromFile(
<< tmp_file_buffer.getError().message();
return Status(ss.str());
}
- file_buffer = std::move(tmp_file_buffer.get());
+ file_buffer = move(tmp_file_buffer.get());
}
COUNTER_ADD(module_bitcode_size_, file_buffer->getBufferSize());
- return LoadModuleFromMemory(std::move(file_buffer), file, module);
+ return LoadModuleFromMemory(move(file_buffer), file, module);
}
Status LlvmCodeGen::LoadModuleFromMemory(unique_ptr<llvm::MemoryBuffer> module_ir_buf,
@@ -284,14 +284,16 @@ Status LlvmCodeGen::LoadModuleFromMemory(unique_ptr<llvm::MemoryBuffer> module_i
DCHECK(!module_name.empty());
SCOPED_TIMER(prepare_module_timer_);
COUNTER_ADD(module_bitcode_size_, module_ir_buf->getMemBufferRef().getBufferSize());
- llvm::ErrorOr<unique_ptr<llvm::Module>> tmp_module =
- getLazyBitcodeModule(std::move(module_ir_buf), context(), false);
- if (!tmp_module) {
- string diagnostic_err = diagnostic_handler_.GetErrorString();
- return Status(diagnostic_err);
+ llvm::Expected<unique_ptr<llvm::Module>> tmp_module =
+ getOwningLazyBitcodeModule(move(module_ir_buf), context());
+ if (llvm::Error err = tmp_module.takeError()) {
+ string err_string;
+ llvm::handleAllErrors(
+ move(err), [&](llvm::ErrorInfoBase& eib) { err_string = eib.message(); });
+ return Status(err_string);
}
- *module = std::move(tmp_module.get());
+ *module = move(tmp_module.get());
// We never run global constructors or destructors so let's strip them out for all
// modules when we load them.
@@ -324,7 +326,7 @@ Status LlvmCodeGen::LinkModuleFromLocalFs(const string& file) {
}
}
- bool error = llvm::Linker::linkModules(*module_, std::move(new_module));
+ bool error = llvm::Linker::linkModules(*module_, move(new_module));
string diagnostic_err = diagnostic_handler_.GetErrorString();
if (error) {
stringstream ss;
@@ -399,7 +401,7 @@ Status LlvmCodeGen::Init(unique_ptr<llvm::Module> module) {
opt_level = llvm::CodeGenOpt::None;
#endif
module_ = module.get();
- llvm::EngineBuilder builder(std::move(module));
+ llvm::EngineBuilder builder(move(module));
builder.setEngineKind(llvm::EngineKind::JIT);
builder.setOptLevel(opt_level);
unique_ptr<ImpalaMCJITMemoryManager> memory_manager(new ImpalaMCJITMemoryManager);
@@ -622,10 +624,13 @@ Status LlvmCodeGen::MaterializeFunctionHelper(llvm::Function* fn) {
DCHECK(!is_compiled_);
if (fn->isIntrinsic() || !fn->isMaterializable()) return Status::OK();
- std::error_code err = module_->materialize(fn);
+ llvm::Error err = module_->materialize(fn);
if (UNLIKELY(err)) {
+ string err_string;
+ llvm::handleAllErrors(
+ move(err), [&](llvm::ErrorInfoBase& eib) { err_string = eib.message(); });
return Status(Substitute("Failed to materialize $0: $1",
- fn->getName().str(), err.message()));
+ fn->getName().str(), err_string));
}
// Materialized functions are marked as not materializable by LLVM.
@@ -723,8 +728,8 @@ bool LlvmCodeGen::VerifyFunction(llvm::Function* fn) {
if (is_corrupt_) {
string fn_name = fn->getName(); // llvm has some fancy operator overloading
- LOG(ERROR) << "Function corrupt: " << fn_name;
- fn->dump();
+ LOG(ERROR) << "Function corrupt: " << fn_name <<"\nFunction Dump: "
+ << LlvmCodeGen::Print(fn);
return false;
}
return true;
@@ -783,8 +788,8 @@ llvm::Function* LlvmCodeGen::FnPrototype::GeneratePrototype(
return fn;
}
-Status LlvmCodeGen::LoadFunction(const TFunction& fn, const std::string& symbol,
- const ColumnType* return_type, const std::vector<ColumnType>& arg_types,
+Status LlvmCodeGen::LoadFunction(const TFunction& fn, const string& symbol,
+ const ColumnType* return_type, const vector<ColumnType>& arg_types,
int num_fixed_args, bool has_varargs, llvm::Function** llvm_fn,
LibCacheEntry** cache_entry) {
DCHECK_GE(arg_types.size(), num_fixed_args);
@@ -1011,15 +1016,21 @@ llvm::Function* LlvmCodeGen::FinalizeFunction(llvm::Function* function) {
return NULL;
}
finalized_functions_.insert(function);
- if (FLAGS_dump_ir) function->dump();
+ if (FLAGS_dump_ir) {
+ string fn_name = function->getName();
+ LOG(INFO) << "Dump of Function "<< fn_name << ": " << LlvmCodeGen::Print(function);
+ }
return function;
}
Status LlvmCodeGen::MaterializeModule() {
- std::error_code err = module_->materializeAll();
+ llvm::Error err = module_->materializeAll();
if (UNLIKELY(err)) {
+ string err_string;
+ llvm::handleAllErrors(
+ move(err), [&](llvm::ErrorInfoBase& eib) { err_string = eib.message(); });
return Status(Substitute("Failed to materialize module $0: $1",
- module_->getName().str(), err.message()));
+ module_->getName().str(), err_string));
}
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/exprs/aggregate-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/aggregate-functions-ir.cc b/be/src/exprs/aggregate-functions-ir.cc
index 9af563e..3bf2867 100644
--- a/be/src/exprs/aggregate-functions-ir.cc
+++ b/be/src/exprs/aggregate-functions-ir.cc
@@ -2060,8 +2060,6 @@ template void AggregateFunctions::UpdateVal<FloatVal>(
FunctionContext*, const FloatVal& src, FloatVal* dst);
template void AggregateFunctions::UpdateVal<DoubleVal>(
FunctionContext*, const DoubleVal& src, DoubleVal* dst);
-template void AggregateFunctions::UpdateVal<StringVal>(
- FunctionContext*, const StringVal& src, StringVal* dst);
template void AggregateFunctions::UpdateVal<TimestampVal>(
FunctionContext*, const TimestampVal& src, TimestampVal* dst);
template void AggregateFunctions::UpdateVal<DecimalVal>(
@@ -2112,14 +2110,6 @@ template void AggregateFunctions::Min<IntVal>(
FunctionContext*, const IntVal& src, IntVal* dst);
template void AggregateFunctions::Min<BigIntVal>(
FunctionContext*, const BigIntVal& src, BigIntVal* dst);
-template void AggregateFunctions::Min<FloatVal>(
- FunctionContext*, const FloatVal& src, FloatVal* dst);
-template void AggregateFunctions::Min<DoubleVal>(
- FunctionContext*, const DoubleVal& src, DoubleVal* dst);
-template void AggregateFunctions::Min<StringVal>(
- FunctionContext*, const StringVal& src, StringVal* dst);
-template void AggregateFunctions::Min<DecimalVal>(
- FunctionContext*, const DecimalVal& src, DecimalVal* dst);
template void AggregateFunctions::Max<BooleanVal>(
FunctionContext*, const BooleanVal& src, BooleanVal* dst);
@@ -2131,14 +2121,6 @@ template void AggregateFunctions::Max<IntVal>(
FunctionContext*, const IntVal& src, IntVal* dst);
template void AggregateFunctions::Max<BigIntVal>(
FunctionContext*, const BigIntVal& src, BigIntVal* dst);
-template void AggregateFunctions::Max<FloatVal>(
- FunctionContext*, const FloatVal& src, FloatVal* dst);
-template void AggregateFunctions::Max<DoubleVal>(
- FunctionContext*, const DoubleVal& src, DoubleVal* dst);
-template void AggregateFunctions::Max<StringVal>(
- FunctionContext*, const StringVal& src, StringVal* dst);
-template void AggregateFunctions::Max<DecimalVal>(
- FunctionContext*, const DecimalVal& src, DecimalVal* dst);
template void AggregateFunctions::PcUpdate(
FunctionContext*, const BooleanVal&, StringVal*);
@@ -2347,8 +2329,6 @@ template void AggregateFunctions::HllUpdate(
FunctionContext*, const StringVal&, StringVal*);
template void AggregateFunctions::HllUpdate(
FunctionContext*, const TimestampVal&, StringVal*);
-template void AggregateFunctions::HllUpdate(
- FunctionContext*, const DecimalVal&, StringVal*);
template void AggregateFunctions::SampledNdvUpdate(
FunctionContext*, const BooleanVal&, const DoubleVal&, StringVal*);
@@ -2398,8 +2378,6 @@ template void AggregateFunctions::LastValRemove<FloatVal>(
FunctionContext*, const FloatVal& src, FloatVal* dst);
template void AggregateFunctions::LastValRemove<DoubleVal>(
FunctionContext*, const DoubleVal& src, DoubleVal* dst);
-template void AggregateFunctions::LastValRemove<StringVal>(
- FunctionContext*, const StringVal& src, StringVal* dst);
template void AggregateFunctions::LastValRemove<TimestampVal>(
FunctionContext*, const TimestampVal& src, TimestampVal* dst);
template void AggregateFunctions::LastValRemove<DecimalVal>(
@@ -2482,8 +2460,6 @@ template FloatVal AggregateFunctions::LastValIgnoreNullsGetValue<FloatVal>(
FunctionContext*, const StringVal&);
template DoubleVal AggregateFunctions::LastValIgnoreNullsGetValue<DoubleVal>(
FunctionContext*, const StringVal&);
-template StringVal AggregateFunctions::LastValIgnoreNullsGetValue<StringVal>(
- FunctionContext*, const StringVal&);
template TimestampVal AggregateFunctions::LastValIgnoreNullsGetValue<TimestampVal>(
FunctionContext*, const StringVal&);
template DecimalVal AggregateFunctions::LastValIgnoreNullsGetValue<DecimalVal>(
@@ -2503,8 +2479,6 @@ template FloatVal AggregateFunctions::LastValIgnoreNullsFinalize<FloatVal>(
FunctionContext*, const StringVal&);
template DoubleVal AggregateFunctions::LastValIgnoreNullsFinalize<DoubleVal>(
FunctionContext*, const StringVal&);
-template StringVal AggregateFunctions::LastValIgnoreNullsFinalize<StringVal>(
- FunctionContext*, const StringVal&);
template TimestampVal AggregateFunctions::LastValIgnoreNullsFinalize<TimestampVal>(
FunctionContext*, const StringVal&);
template DecimalVal AggregateFunctions::LastValIgnoreNullsFinalize<DecimalVal>(
@@ -2524,8 +2498,6 @@ template void AggregateFunctions::FirstValUpdate<FloatVal>(
FunctionContext*, const FloatVal& src, FloatVal* dst);
template void AggregateFunctions::FirstValUpdate<DoubleVal>(
FunctionContext*, const DoubleVal& src, DoubleVal* dst);
-template void AggregateFunctions::FirstValUpdate<StringVal>(
- FunctionContext*, const StringVal& src, StringVal* dst);
template void AggregateFunctions::FirstValUpdate<TimestampVal>(
FunctionContext*, const TimestampVal& src, TimestampVal* dst);
template void AggregateFunctions::FirstValUpdate<DecimalVal>(
@@ -2566,8 +2538,6 @@ template void AggregateFunctions::FirstValIgnoreNullsUpdate<FloatVal>(
FunctionContext*, const FloatVal& src, FloatVal* dst);
template void AggregateFunctions::FirstValIgnoreNullsUpdate<DoubleVal>(
FunctionContext*, const DoubleVal& src, DoubleVal* dst);
-template void AggregateFunctions::FirstValIgnoreNullsUpdate<StringVal>(
- FunctionContext*, const StringVal& src, StringVal* dst);
template void AggregateFunctions::FirstValIgnoreNullsUpdate<TimestampVal>(
FunctionContext*, const TimestampVal& src, TimestampVal* dst);
template void AggregateFunctions::FirstValIgnoreNullsUpdate<DecimalVal>(
@@ -2587,8 +2557,6 @@ template void AggregateFunctions::OffsetFnInit<FloatVal>(
FunctionContext*, FloatVal*);
template void AggregateFunctions::OffsetFnInit<DoubleVal>(
FunctionContext*, DoubleVal*);
-template void AggregateFunctions::OffsetFnInit<StringVal>(
- FunctionContext*, StringVal*);
template void AggregateFunctions::OffsetFnInit<TimestampVal>(
FunctionContext*, TimestampVal*);
template void AggregateFunctions::OffsetFnInit<DecimalVal>(
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/rpc/impala-service-pool.h
----------------------------------------------------------------------
diff --git a/be/src/rpc/impala-service-pool.h b/be/src/rpc/impala-service-pool.h
index 9d34366..910ac53 100644
--- a/be/src/rpc/impala-service-pool.h
+++ b/be/src/rpc/impala-service-pool.h
@@ -52,10 +52,10 @@ class ImpalaServicePool : public kudu::rpc::RpcService {
virtual ~ImpalaServicePool();
/// Start up the thread pool.
- virtual Status Init(int num_threads);
+ Status Init(int num_threads);
/// Shut down the queue and the thread pool.
- virtual void Shutdown();
+ void Shutdown();
virtual kudu::rpc::RpcMethodInfo* LookupMethod(const kudu::rpc::RemoteMethod& method)
override;
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/runtime/bufferpool/buffer-pool-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/bufferpool/buffer-pool-test.cc b/be/src/runtime/bufferpool/buffer-pool-test.cc
index d6547d2..5337bb4 100644
--- a/be/src/runtime/bufferpool/buffer-pool-test.cc
+++ b/be/src/runtime/bufferpool/buffer-pool-test.cc
@@ -1960,7 +1960,7 @@ void BufferPoolTest::TestRandomInternalMulti(
}
AtomicInt32 stop_maintenance(0);
- thread* maintenance_thread = new thread([this, &pool, &stop_maintenance]() {
+ thread* maintenance_thread = new thread([&pool, &stop_maintenance]() {
while (stop_maintenance.Load() == 0) {
pool.Maintenance();
SleepForMs(50);
@@ -2171,7 +2171,7 @@ TEST_F(BufferPoolTest, ConcurrentBufferOperations) {
// Allocate threads allocate buffers whenever able and enqueue them.
for (int i = 0; i < ALLOCATE_THREADS; ++i) {
allocate_threads.add_thread(new thread([&] {
- for (int i = 0; i < NUM_ALLOCATIONS_PER_THREAD; ++i) {
+ for (int j = 0; j < NUM_ALLOCATIONS_PER_THREAD; ++j) {
// Try to deduct reservation.
while (true) {
int64_t val = available_reservation.Load();
@@ -2182,7 +2182,7 @@ TEST_F(BufferPoolTest, ConcurrentBufferOperations) {
}
BufferHandle buffer;
ASSERT_OK(pool.AllocateBuffer(&client, TEST_BUFFER_LEN, &buffer));
- uint8_t first_byte = static_cast<uint8_t>(i % 256);
+ uint8_t first_byte = static_cast<uint8_t>(j % 256);
buffer.data()[0] = first_byte;
delete_queue.BlockingPut(pair<uint8_t, BufferHandle>(first_byte, move(buffer)));
}
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/runtime/data-stream-sender.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/data-stream-sender.cc b/be/src/runtime/data-stream-sender.cc
index a744929..f68788e 100644
--- a/be/src/runtime/data-stream-sender.cc
+++ b/be/src/runtime/data-stream-sender.cc
@@ -337,7 +337,6 @@ DataStreamSender::DataStreamSender(int sender_id, const RowDescriptor* row_desc,
partition_type_(sink.output_partition.type),
current_channel_idx_(0),
flushed_(false),
- closed_(false),
current_thrift_batch_(&thrift_batch1_),
serialize_batch_timer_(NULL),
thrift_transmit_timer_(NULL),
@@ -534,7 +533,6 @@ void DataStreamSender::Close(RuntimeState* state) {
ScalarExprEvaluator::Close(partition_expr_evals_, state);
ScalarExpr::Close(partition_exprs_);
DataSink::Close(state);
- closed_ = true;
}
Status DataStreamSender::SerializeBatch(
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/runtime/data-stream-sender.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/data-stream-sender.h b/be/src/runtime/data-stream-sender.h
index 992e9c9..37b9417 100644
--- a/be/src/runtime/data-stream-sender.h
+++ b/be/src/runtime/data-stream-sender.h
@@ -116,9 +116,6 @@ class DataStreamSender : public DataSink {
/// Not valid to call Send() anymore.
bool flushed_;
- /// If true, this sender has been closed. Not valid to call Send() anymore.
- bool closed_;
-
/// serialized batches for broadcasting; we need two so we can write
/// one while the other one is still being sent
TRowBatch thrift_batch1_;
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/runtime/krpc-data-stream-sender.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/krpc-data-stream-sender.cc b/be/src/runtime/krpc-data-stream-sender.cc
index e6ff0a2..bc6a07d 100644
--- a/be/src/runtime/krpc-data-stream-sender.cc
+++ b/be/src/runtime/krpc-data-stream-sender.cc
@@ -756,7 +756,6 @@ void KrpcDataStreamSender::Close(RuntimeState* state) {
ScalarExprEvaluator::Close(partition_expr_evals_, state);
ScalarExpr::Close(partition_exprs_);
DataSink::Close(state);
- closed_ = true;
}
Status KrpcDataStreamSender::SerializeBatch(
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/runtime/krpc-data-stream-sender.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/krpc-data-stream-sender.h b/be/src/runtime/krpc-data-stream-sender.h
index 0c3a32e..bc18574 100644
--- a/be/src/runtime/krpc-data-stream-sender.h
+++ b/be/src/runtime/krpc-data-stream-sender.h
@@ -140,9 +140,6 @@ class KrpcDataStreamSender : public DataSink {
/// Not valid to call Send() anymore.
bool flushed_ = false;
- /// If true, this sender has been closed. Not valid to call Send() anymore.
- bool closed_ = false;
-
/// List of all channels. One for each destination.
std::vector<Channel*> channels_;
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/runtime/row-batch-serialize-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/row-batch-serialize-test.cc b/be/src/runtime/row-batch-serialize-test.cc
index b6a0009..e765025 100644
--- a/be/src/runtime/row-batch-serialize-test.cc
+++ b/be/src/runtime/row-batch-serialize-test.cc
@@ -716,7 +716,8 @@ TEST_F(RowBatchSerializeTest, DedupPathologicalFull) {
cv.ptr = pool->Allocate(array_item_desc->byte_size());
memset(cv.ptr, 0, array_item_desc->byte_size());
cv.num_tuples = 1;
- StringValue huge_string_value((char*)huge_string.data(), huge_string_size);
+ StringValue huge_string_value(
+ const_cast<char*>(huge_string.data()), huge_string_size);
RawValue::Write(&huge_string_value, reinterpret_cast<Tuple*>(cv.ptr),
string_slot_desc, NULL);
RawValue::Write(&cv, tuple, array_slot_desc, NULL);
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/runtime/row-batch.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/row-batch.cc b/be/src/runtime/row-batch.cc
index 87c0f2c..3581bd9 100644
--- a/be/src/runtime/row-batch.cc
+++ b/be/src/runtime/row-batch.cc
@@ -309,8 +309,10 @@ Status RowBatch::Serialize(bool full_dedup, vector<int32_t>* tuple_offsets,
if (compression_scratch_.size() < compressed_size) {
compression_scratch_.resize(compressed_size);
}
- uint8_t* input = (uint8_t*)tuple_data->c_str();
- uint8_t* compressed_output = (uint8_t*)compression_scratch_.c_str();
+ uint8_t* input =
+ const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(tuple_data->c_str()));
+ uint8_t* compressed_output = const_cast<uint8_t*>(
+ reinterpret_cast<const uint8_t*>(compression_scratch_.c_str()));
RETURN_IF_ERROR(
compressor.ProcessBlock(true, size, input, &compressed_size, &compressed_output));
if (LIKELY(compressed_size < size)) {
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/runtime/row-batch.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/row-batch.h b/be/src/runtime/row-batch.h
index 1b3778e..270318a 100644
--- a/be/src/runtime/row-batch.h
+++ b/be/src/runtime/row-batch.h
@@ -56,14 +56,17 @@ class OutboundRowBatch {
/// Returns the serialized tuple offsets' vector as a kudu::Slice.
/// The tuple offsets vector is sent as KRPC sidecar.
kudu::Slice TupleOffsetsAsSlice() const {
- return kudu::Slice((uint8_t*)tuple_offsets_.data(),
+ return kudu::Slice(
+ const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(tuple_offsets_.data())),
tuple_offsets_.size() * sizeof(tuple_offsets_[0]));
}
/// Returns the serialized tuple data's buffer as a kudu::Slice.
/// The tuple data is sent as KRPC sidecar.
kudu::Slice TupleDataAsSlice() const {
- return kudu::Slice((uint8_t*)tuple_data_.data(), tuple_data_.length());
+ return kudu::Slice(
+ const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(tuple_data_.data())),
+ tuple_data_.length());
}
/// Returns true if the header has been intialized and ready to be sent.
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/runtime/types.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/types.cc b/be/src/runtime/types.cc
index c18c66e..ab8b8ec 100644
--- a/be/src/runtime/types.cc
+++ b/be/src/runtime/types.cc
@@ -366,7 +366,7 @@ llvm::ConstantStruct* ColumnType::ToIR(LlvmCodeGen* codegen) const {
return llvm::cast<llvm::ConstantStruct>(
llvm::ConstantStruct::get(column_type_type, type_field, len_field, precision_field,
- scale_field, children_field, field_names_field, NULL));
+ scale_field, children_field, field_names_field));
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/service/fe-support.cc
----------------------------------------------------------------------
diff --git a/be/src/service/fe-support.cc b/be/src/service/fe-support.cc
index abb4acf..a8906a0 100644
--- a/be/src/service/fe-support.cc
+++ b/be/src/service/fe-support.cc
@@ -559,48 +559,52 @@ namespace impala {
static JNINativeMethod native_methods[] = {
{
- (char*)"NativeFeTestInit", (char*)"()V",
+ const_cast<char*>("NativeFeTestInit"), const_cast<char*>("()V"),
(void*)::Java_org_apache_impala_service_FeSupport_NativeFeTestInit
},
{
- (char*)"NativeEvalExprsWithoutRow", (char*)"([B[B)[B",
+ const_cast<char*>("NativeEvalExprsWithoutRow"), const_cast<char*>("([B[B)[B"),
(void*)::Java_org_apache_impala_service_FeSupport_NativeEvalExprsWithoutRow
},
{
- (char*)"NativeCacheJar", (char*)"([B)[B",
+ const_cast<char*>("NativeCacheJar"), const_cast<char*>("([B)[B"),
(void*)::Java_org_apache_impala_service_FeSupport_NativeCacheJar
},
{
- (char*)"NativeLookupSymbol", (char*)"([B)[B",
+ const_cast<char*>("NativeLookupSymbol"), const_cast<char*>("([B)[B"),
(void*)::Java_org_apache_impala_service_FeSupport_NativeLookupSymbol
},
{
- (char*)"NativePrioritizeLoad", (char*)"([B)[B",
+ const_cast<char*>("NativePrioritizeLoad"), const_cast<char*>("([B)[B"),
(void*)::Java_org_apache_impala_service_FeSupport_NativePrioritizeLoad
},
{
- (char*)"NativeParseQueryOptions", (char*)"(Ljava/lang/String;[B)[B",
+ const_cast<char*>("NativeParseQueryOptions"),
+ const_cast<char*>("(Ljava/lang/String;[B)[B"),
(void*)::Java_org_apache_impala_service_FeSupport_NativeParseQueryOptions
},
{
- (char*)"NativeAddPendingTopicItem", (char*)"(JLjava/lang/String;J[BZ)Z",
+ const_cast<char*>("NativeAddPendingTopicItem"),
+ const_cast<char*>("(JLjava/lang/String;J[BZ)Z"),
(void*)::Java_org_apache_impala_service_FeSupport_NativeAddPendingTopicItem
},
{
- (char*)"NativeGetNextCatalogObjectUpdate",
- (char*)"(J)Lorg/apache/impala/common/Pair;",
+ const_cast<char*>("NativeGetNextCatalogObjectUpdate"),
+ const_cast<char*>("(J)Lorg/apache/impala/common/Pair;"),
(void*)::Java_org_apache_impala_service_FeSupport_NativeGetNextCatalogObjectUpdate
},
{
- (char*)"NativeLibCacheSetNeedsRefresh", (char*)"(Ljava/lang/String;)Z",
+ const_cast<char*>("NativeLibCacheSetNeedsRefresh"),
+ const_cast<char*>("(Ljava/lang/String;)Z"),
(void*)::Java_org_apache_impala_service_FeSupport_NativeLibCacheSetNeedsRefresh
},
{
- (char*)"NativeLibCacheRemoveEntry", (char*)"(Ljava/lang/String;)Z",
+ const_cast<char*>("NativeLibCacheRemoveEntry"),
+ const_cast<char*>("(Ljava/lang/String;)Z"),
(void*)::Java_org_apache_impala_service_FeSupport_NativeLibCacheRemoveEntry
},
{
- (char*)"MinLogSpaceForBloomFilter", (char*)"(JD)I",
+ const_cast<char*>("MinLogSpaceForBloomFilter"), const_cast<char*>("(JD)I"),
(void*)::Java_org_apache_impala_service_FeSupport_MinLogSpaceForBloomFilter
},
};
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/thirdparty/squeasel/squeasel.c
----------------------------------------------------------------------
diff --git a/be/src/thirdparty/squeasel/squeasel.c b/be/src/thirdparty/squeasel/squeasel.c
index d461b24..2149497 100644
--- a/be/src/thirdparty/squeasel/squeasel.c
+++ b/be/src/thirdparty/squeasel/squeasel.c
@@ -1057,6 +1057,7 @@ static int alloc_vprintf2(char **buf, const char *fmt, va_list ap) {
if (!*buf) break;
va_copy(ap_copy, ap);
len = vsnprintf(*buf, size, fmt, ap_copy);
+ va_end(ap_copy);
}
return len;
@@ -1076,12 +1077,14 @@ static int alloc_vprintf(char **buf, size_t size, const char *fmt, va_list ap) {
// On second pass, actually print the message.
va_copy(ap_copy, ap);
len = vsnprintf(NULL, 0, fmt, ap_copy);
+ va_end(ap_copy);
if (len < 0) {
// C runtime is not standard compliant, vsnprintf() returned -1.
// Switch to alternative code path that uses incremental allocations.
va_copy(ap_copy, ap);
len = alloc_vprintf2(buf, fmt, ap);
+ va_end(ap_copy);
} else if (len > (int) size &&
(size = len + 1) > 0 &&
(*buf = (char *) malloc(size)) == NULL) {
@@ -1089,6 +1092,7 @@ static int alloc_vprintf(char **buf, size_t size, const char *fmt, va_list ap) {
} else {
va_copy(ap_copy, ap);
vsnprintf(*buf, size, fmt, ap_copy);
+ va_end(ap_copy);
}
return len;
@@ -1111,7 +1115,9 @@ int sq_vprintf(struct sq_connection *conn, const char *fmt, va_list ap) {
int sq_printf(struct sq_connection *conn, const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
- return sq_vprintf(conn, fmt, ap);
+ int ret_val = sq_vprintf(conn, fmt, ap);
+ va_end(ap);
+ return ret_val;
}
int sq_url_decode(const char *src, int src_len, char *dst,
@@ -4476,6 +4482,7 @@ struct sq_connection *sq_download(const char *host, int port, int use_ssl,
sq_close_connection(conn);
conn = NULL;
}
+ va_end(ap);
return conn;
}
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/transport/TSasl.cpp
----------------------------------------------------------------------
diff --git a/be/src/transport/TSasl.cpp b/be/src/transport/TSasl.cpp
index 7b4d94d..a1e4a39 100644
--- a/be/src/transport/TSasl.cpp
+++ b/be/src/transport/TSasl.cpp
@@ -54,8 +54,8 @@ uint8_t* TSasl::unwrap(const uint8_t* incoming,
uint8_t* output;
int result;
- result = sasl_decode(conn,
- (const char*)incoming, len, (const char**)&output, &outputlen);
+ result = sasl_decode(conn, reinterpret_cast<const char*>(incoming), len,
+ const_cast<const char**>(reinterpret_cast<char**>(&output)), &outputlen);
if (result != SASL_OK) {
throw SaslException(sasl_errdetail(conn));
}
@@ -69,8 +69,8 @@ uint8_t* TSasl::wrap(const uint8_t* outgoing,
uint8_t* output;
int result;
- result = sasl_encode(conn, (const char*)outgoing+offset,
- len, (const char**)&output, &outputlen);
+ result = sasl_encode(conn, reinterpret_cast<const char*>(outgoing) + offset, len,
+ const_cast<const char**>(reinterpret_cast<char**>(&output)), &outputlen);
if (result != SASL_OK) {
throw SaslException(sasl_errdetail(conn));
}
@@ -158,9 +158,10 @@ uint8_t* TSaslClient::evaluateChallengeOrResponse(
result=sasl_client_start(conn,
mechList.c_str(),
&client_interact, /* filled in if an interaction is needed */
- (const char**)&out, /* filled in on success */
+ const_cast<const char**>(
+ reinterpret_cast<char**>(&out)), /* filled in on success */
&outlen, /* filled in on success */
- (const char**)&mechUsing);
+ const_cast<const char**>(&mechUsing));
clientStarted = true;
if (result == SASL_OK || result == SASL_CONTINUE) {
chosenMech = mechUsing;
@@ -168,10 +169,11 @@ uint8_t* TSaslClient::evaluateChallengeOrResponse(
} else {
if (len > 0) {
result=sasl_client_step(conn, /* our context */
- (const char*)challenge, /* the data from the server */
+ reinterpret_cast<const char*>(challenge), /* the data from the server */
len, /* its length */
&client_interact, /* this should be unallocated and NULL */
- (const char**)&out, /* filled in on success */
+ const_cast<const char**>(
+ reinterpret_cast<char**>(&out)), /* filled in on success */
&outlen); /* filled in on success */
} else {
result = SASL_CONTINUE;
@@ -237,11 +239,11 @@ uint8_t* TSaslServer::evaluateChallengeOrResponse(const uint8_t* response,
uint32_t result;
if (!serverStarted) {
- result = sasl_server_start(conn,
- (const char *)response, NULL, 0, (const char **)&out, &outlen);
+ result = sasl_server_start(conn, reinterpret_cast<const char*>(response), NULL, 0,
+ const_cast<const char**>(reinterpret_cast<char**>(&out)), &outlen);
} else {
- result = sasl_server_step(conn,
- (const char*)response, len, (const char**)&out, &outlen);
+ result = sasl_server_step(conn, reinterpret_cast<const char*>(response), len,
+ const_cast<const char**>(reinterpret_cast<char**>(&out)), &outlen);
}
if (result == SASL_OK) {
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/transport/TSaslClientTransport.cpp
----------------------------------------------------------------------
diff --git a/be/src/transport/TSaslClientTransport.cpp b/be/src/transport/TSaslClientTransport.cpp
index 279a966..5d20e63 100644
--- a/be/src/transport/TSaslClientTransport.cpp
+++ b/be/src/transport/TSaslClientTransport.cpp
@@ -65,8 +65,8 @@ void TSaslClientTransport::handleSaslStartMessage() {
}
/* These two calls comprise a single message in the thrift-sasl protocol. */
- sendSaslMessage(TSASL_START,
- (uint8_t*)sasl_->getMechanismName().c_str(),
+ sendSaslMessage(TSASL_START, const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(
+ sasl_->getMechanismName().c_str())),
sasl_->getMechanismName().length(), false);
sendSaslMessage(TSASL_OK, initialResponse, resLength);
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/transport/TSaslTransport.cpp
----------------------------------------------------------------------
diff --git a/be/src/transport/TSaslTransport.cpp b/be/src/transport/TSaslTransport.cpp
index 55e57c8..20eaef7 100644
--- a/be/src/transport/TSaslTransport.cpp
+++ b/be/src/transport/TSaslTransport.cpp
@@ -225,7 +225,7 @@ namespace apache { namespace thrift { namespace transport {
const uint8_t* newBuf;
if (shouldWrap_) {
- newBuf = sasl_->wrap((uint8_t*)buf, 0, len, &len);
+ newBuf = sasl_->wrap(const_cast<uint8_t*>(buf), 0, len, &len);
} else {
newBuf = buf;
}
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/udf/udf-test.cc
----------------------------------------------------------------------
diff --git a/be/src/udf/udf-test.cc b/be/src/udf/udf-test.cc
index 0bce70f..d16fb08 100644
--- a/be/src/udf/udf-test.cc
+++ b/be/src/udf/udf-test.cc
@@ -136,7 +136,8 @@ IntVal ValidateMem(FunctionContext* context) {
}
StringVal TimeToString(FunctionContext* context, const TimestampVal& time) {
- ptime t(*(date*)&time.date);
+ ptime t(*const_cast<date*>(reinterpret_cast<const date*>(&time.date)));
+ // ptime t(*(date*)&time.date); is this conversion correct?
t += nanoseconds(time.time_of_day);
stringstream ss;
ss << to_iso_extended_string(t.date()) << " " << to_simple_string(t.time_of_day());
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/be/src/udf/udf.h
----------------------------------------------------------------------
diff --git a/be/src/udf/udf.h b/be/src/udf/udf.h
index 3d04115..26e1f45 100644
--- a/be/src/udf/udf.h
+++ b/be/src/udf/udf.h
@@ -599,7 +599,9 @@ struct StringVal : public AnyVal {
/// Construct a StringVal from NULL-terminated c-string. Note: this does not make a
/// copy of ptr so the underlying string must exist as long as this StringVal does.
- StringVal(const char* ptr) : len(strlen(ptr)), ptr((uint8_t*)ptr) {}
+ StringVal(const char* ptr)
+ : len(strlen(ptr)),
+ ptr(const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(ptr))) {}
/// Creates a StringVal, allocating a new buffer with 'len'. This should
/// be used to return StringVal objects in UDF/UDAs that need to allocate new
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/bin/bootstrap_toolchain.py
----------------------------------------------------------------------
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index 8cee665..a09c905 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -431,7 +431,7 @@ if __name__ == "__main__":
"flatbuffers", "gcc", "gflags", "glog", "gperftools", "gtest", "libev",
"lz4", "openldap", "openssl", "protobuf",
"rapidjson", "re2", "snappy", "thrift", "tpc-h", "tpc-ds", "zlib"])
- packages.insert(0, Package("llvm", "3.9.1-asserts"))
+ packages.insert(0, Package("llvm", "5.0.1-asserts"))
bootstrap(toolchain_root, packages)
# Download the CDH components if necessary.
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index a70149a..0e9d253 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -72,7 +72,7 @@ fi
# moving to a different build of the toolchain, e.g. when a version is bumped or a
# compile option is changed. The build id can be found in the output of the toolchain
# build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=53-d95bb7f778
+export IMPALA_TOOLCHAIN_BUILD_ID=88-8e37339d45
# Versions of toolchain dependencies.
# -----------------------------------
export IMPALA_AVRO_VERSION=1.7.4-p4
@@ -105,14 +105,14 @@ export IMPALA_GTEST_VERSION=1.6.0
unset IMPALA_GTEST_URL
export IMPALA_LIBEV_VERSION=4.20
unset IMPALA_LIBEV_URL
-export IMPALA_LLVM_VERSION=3.9.1
+export IMPALA_LLVM_VERSION=5.0.1
unset IMPALA_LLVM_URL
-export IMPALA_LLVM_ASAN_VERSION=3.9.1
+export IMPALA_LLVM_ASAN_VERSION=5.0.1
unset IMPALA_LLVM_ASAN_URL
# Debug builds should use the release+asserts build to get additional coverage.
# Don't use the LLVM debug build because the binaries are too large to distribute.
-export IMPALA_LLVM_DEBUG_VERSION=3.9.1-asserts
+export IMPALA_LLVM_DEBUG_VERSION=5.0.1-asserts
unset IMPALA_LLVM_DEBUG_URL
export IMPALA_LZ4_VERSION=1.7.5
unset IMPALA_LZ4_URL
http://git-wip-us.apache.org/repos/asf/impala/blob/4a39e7c2/testdata/workloads/functional-query/queries/QueryTest/udf-errors.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/udf-errors.test b/testdata/workloads/functional-query/queries/QueryTest/udf-errors.test
index 81e459a..fb91070 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/udf-errors.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/udf-errors.test
@@ -24,6 +24,7 @@ create function if not exists foo (string, string) returns string location
'$FILESYSTEM_PREFIX/test-warehouse/$DATABASE_bad_udf.ll' symbol='MyAwesomeUdf';
---- CATCH
Could not load binary: $FILESYSTEM_PREFIX/test-warehouse/$DATABASE_bad_udf.ll
+Invalid bitcode signature
====
---- QUERY
# This test is run with codegen disabled. Interpretation only handles up to 20 arguments.