You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by he...@apache.org on 2017/03/01 17:28:40 UTC
[1/6] incubator-impala git commit: IMPALA-4982: Add parquet stats test
Repository: incubator-impala
Updated Branches:
refs/heads/master 674d31325 -> 1609585dc
IMPALA-4982: Add parquet stats test
IMPALA-2328 added support for skipping row groups based on
parquet::Statistics. This change adds a test for root-level
scalar columns of parquet files with nested types.
Change-Id: If81c8a1ecea937794885d4e5e7bf765bd238f5fb
Reviewed-on: http://gerrit.cloudera.org:8080/6130
Reviewed-by: Lars Volker <lv...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/1417d764
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/1417d764
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/1417d764
Branch: refs/heads/master
Commit: 1417d764e489acf8b93f56787aa988d1e94768ce
Parents: 674d313
Author: Lars Volker <lv...@cloudera.com>
Authored: Thu Feb 23 14:01:52 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Mar 1 02:00:19 2017 +0000
----------------------------------------------------------------------
.../QueryTest/nested-types-parquet-stats.test | 30 ++++++++++++++++++++
tests/query_test/test_nested_types.py | 7 +++++
2 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1417d764/testdata/workloads/functional-query/queries/QueryTest/nested-types-parquet-stats.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-parquet-stats.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-parquet-stats.test
new file mode 100644
index 0000000..c8ba303
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-parquet-stats.test
@@ -0,0 +1,30 @@
+====
+---- QUERY
+# Filter root-level scalar column in file with nested types.
+select count(*) from functional_parquet.complextypestbl where id < 1
+---- RESULTS
+0
+---- RUNTIME_PROFILE
+row_regex: .*NumRowGroups: 2 .*
+row_regex: .*NumStatsFilteredRowGroups: 2 .*
+====
+---- QUERY
+# Filter root-level scalar column in file with nested types.
+select id, int_array.item
+from functional_parquet.complextypestbl, complextypestbl.int_array
+where id < 0;
+---- RESULTS
+---- RUNTIME_PROFILE
+row_regex: .*NumRowGroups: 2 .*
+row_regex: .*NumStatsFilteredRowGroups: 2 .*
+====
+---- QUERY
+# Nested columns do not support stats based filtering.
+select id, int_array.item
+from functional_parquet.complextypestbl, functional_parquet.complextypestbl.int_array
+where int_array.item < -1;
+---- RESULTS
+---- RUNTIME_PROFILE
+row_regex: .*NumRowGroups: 2 .*
+row_regex: .*NumStatsFilteredRowGroups: 0 .*
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1417d764/tests/query_test/test_nested_types.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py
index 59fcced..1c65181 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -75,6 +75,13 @@ class TestNestedTypes(ImpalaTestSuite):
"""Queries over the larger nested TPCH dataset."""
self.run_test_case('QueryTest/nested-types-tpch', vector)
+ def test_parquet_stats(self, vector):
+ """Queries that test evaluation of Parquet row group statistics."""
+ # The test makes assumptions about the number of row groups that are processed and
+ # skipped inside a fragment, so we ensure that the tests run in a single fragment.
+ vector.get_value('exec_option')['num_nodes'] = 1
+ self.run_test_case('QueryTest/nested-types-parquet-stats', vector)
+
@SkipIfOldAggsJoins.nested_types
class TestParquetArrayEncodings(ImpalaTestSuite):
TESTFILE_DIR = os.path.join(os.environ['IMPALA_HOME'],
[6/6] incubator-impala git commit: IMPALA-4995: Fix integer overflow
in TopNNode::PrepareForOutput
Posted by he...@apache.org.
IMPALA-4995: Fix integer overflow in TopNNode::PrepareForOutput
To test this, Matt Mulder ran the failing query from IMPALA-4995 on a
private cluster and it did not crash. However the query did not finish
within several hours. We should switch to using the Sorter for large
TopN queries, as tracked by IMPALA-5004.
Change-Id: I5048ec67d8f086346220d56e027e6583fbb5ddad
Reviewed-on: http://gerrit.cloudera.org:8080/6171
Reviewed-by: Lars Volker <lv...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/1609585d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/1609585d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/1609585d
Branch: refs/heads/master
Commit: 1609585dc5e9efca8c2fcab4a08474304e04302c
Parents: abe860c
Author: Lars Volker <lv...@cloudera.com>
Authored: Mon Feb 27 15:18:37 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Mar 1 07:50:00 2017 +0000
----------------------------------------------------------------------
be/src/exec/topn-node.cc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1609585d/be/src/exec/topn-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/topn-node.cc b/be/src/exec/topn-node.cc
index a7f11bd..b79191f 100644
--- a/be/src/exec/topn-node.cc
+++ b/be/src/exec/topn-node.cc
@@ -221,7 +221,7 @@ void TopNNode::Close(RuntimeState* state) {
// Reverse the order of the tuples in the priority queue
void TopNNode::PrepareForOutput() {
sorted_top_n_.resize(priority_queue_->size());
- int index = sorted_top_n_.size() - 1;
+ int64_t index = sorted_top_n_.size() - 1;
while (priority_queue_->size() > 0) {
Tuple* tuple = priority_queue_->top();
[5/6] incubator-impala git commit: IMPALA-4997: Fix overflows in
Sorter::TupleIterator
Posted by he...@apache.org.
IMPALA-4997: Fix overflows in Sorter::TupleIterator
Various places in Sorter::TupleIterator multiply two int values
(Sorter::Run::block_capacity_ and Sorter::TupleIterator::block_index_)
and assigned the result to an int64_t value
(Sorter::TupleIterator::buffer_start_index_). One such occurrence is in
be/src/runtime/sorter.cc#L1080. This multiplication could overflow for
runs with a large number of rows. Changing one of the operands to
int64_t fixes this.
To test this Matt Mulder ran the failing query from IMPALA-4997 on a
private cluster and it succeeded.
Change-Id: Iea22aa96e0cc86102b60c6e551e9e607cef485c8
Reviewed-on: http://gerrit.cloudera.org:8080/6169
Reviewed-by: Lars Volker <lv...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/abe860ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/abe860ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/abe860ce
Branch: refs/heads/master
Commit: abe860ced09d2bcd87f534e1e4e655f4b17d2aae
Parents: 09f32d4
Author: Lars Volker <lv...@cloudera.com>
Authored: Mon Feb 27 13:29:24 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Mar 1 07:41:42 2017 +0000
----------------------------------------------------------------------
be/src/runtime/sorter.cc | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/abe860ce/be/src/runtime/sorter.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/sorter.cc b/be/src/runtime/sorter.cc
index 5f6bf8b..01554b8 100644
--- a/be/src/runtime/sorter.cc
+++ b/be/src/runtime/sorter.cc
@@ -258,8 +258,10 @@ class Sorter::Run {
/// The size in bytes of the sort tuple.
const int sort_tuple_size_;
- /// Number of tuples per block in a run.
- const int block_capacity_;
+ /// Number of tuples per block in a run. This gets multiplied with
+ /// TupleIterator::block_index_ in various places and to make sure we don't overflow the
+ /// result of that operation we make this int64_t here.
+ const int64_t block_capacity_;
const bool has_var_len_slots_;
[4/6] incubator-impala git commit: IMPALA-4877: fix precedence of
unary -/+
Posted by he...@apache.org.
IMPALA-4877: fix precedence of unary -/+
Currently, expressions such as "-2 / 3" parse as "-(2 / 3)". In
practice this usually doesn't cause differences for most common
expressions. However, it does interact with a heuristic that changes
decimal expression types to double when one operand is non-decimal
(see JIRA). For example, before this fix,
typeof(2.0/3.0) = DECIMAL
typeof(-2.0/3.0) = DOUBLE
because the latter expression parsed as "-1 * (2.0 / 3.0)".
With this fix, both expressions result in a DECIMAL.
Technically this is a compatibility breaking change but it seems
like no one would expect the current behavior so I think we should
fix it. Let me know if you disagree.
Change-Id: I39cf388ae6e37e1b1e12ddea5fd3878c9c2620d1
Reviewed-on: http://gerrit.cloudera.org:8080/6044
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/09f32d40
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/09f32d40
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/09f32d40
Branch: refs/heads/master
Commit: 09f32d406bfc3976b8312a7dcd273b7b0679ffbb
Parents: c848074
Author: Dan Hecht <dh...@cloudera.com>
Authored: Thu Feb 16 16:42:41 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Mar 1 04:18:14 2017 +0000
----------------------------------------------------------------------
be/src/exprs/expr-test.cc | 17 +++++++++++++++++
fe/src/main/cup/sql-parser.cup | 4 ++++
2 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/09f32d40/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 37e816d..e9dbd76 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -1392,6 +1392,18 @@ DecimalTestCase decimal_cases[] = {
{ "cast(111111.1111 as decimal(20, 10)) / cast(.7777 as decimal(38, 38))",
{{ true, 0, 38, 38 },
{ false, 142871429985, 38, 6 }}},
+ { "2.0 / 3.0",
+ {{ false, 6666, 6, 4},
+ { false, 666666, 8, 6}}},
+ { "-2.0 / 3.0",
+ {{ false, -6666, 6, 4},
+ { false, -666666, 8, 6}}},
+ { "2.0 / -3.0",
+ {{ false, -6666, 6, 4},
+ { false, -666666, 8, 6}}},
+ { "-2.0 / -3.0",
+ {{ false, 6666, 6, 4},
+ { false, 666666, 8, 6}}},
// Test modulo operator
{ "cast(1.23 as decimal(8,2)) % cast(1 as decimal(10,3))", {{ false, 230, 9, 3 }}},
{ "cast(1 as decimal(38,0)) % cast(.2 as decimal(38,1))", {{ false, 0, 38, 1 }}},
@@ -4028,12 +4040,17 @@ TEST_F(ExprTest, UnaryOperators) {
TestValue("- -1", TYPE_TINYINT, 1);
TestValue("+-1", TYPE_TINYINT, -1);
TestValue("++1", TYPE_TINYINT, 1);
+ TestValue("~1", TYPE_TINYINT, -2);
TestValue("+cast(1. as float)", TYPE_FLOAT, 1.0f);
TestValue("+cast(1.0 as float)", TYPE_FLOAT, 1.0f);
TestValue("-cast(1.0 as float)", TYPE_DOUBLE, -1.0);
TestValue("1 - - - 1", TYPE_SMALLINT, 0);
+
+ // IMPALA-4877: Verify that unary minus has high precedence and is integrated into
+ // literals.
+ TestValue("-1 & 8", TYPE_TINYINT, 8);
}
// TODO: I think a lot of these casts are not necessary and we should fix this
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/09f32d40/fe/src/main/cup/sql-parser.cup
----------------------------------------------------------------------
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index 867850d..8d0f739 100644
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -273,6 +273,7 @@ terminal
terminal COLON, SEMICOLON, COMMA, DOT, DOTDOTDOT, STAR, LPAREN, RPAREN, LBRACKET,
RBRACKET, DIVIDE, MOD, ADD, SUBTRACT;
+terminal UNARYSIGN; // Placeholder terminal for unary -/+
terminal BITAND, BITOR, BITXOR, BITNOT;
terminal EQUAL, NOT, NOTEQUAL, LESSTHAN, GREATERTHAN;
terminal FACTORIAL; // Placeholder terminal for postfix factorial operator
@@ -513,6 +514,7 @@ precedence left EQUAL, NOTEQUAL, LESSTHAN, GREATERTHAN, KW_FROM, KW_DISTINCT;
precedence left ADD, SUBTRACT;
precedence left STAR, DIVIDE, MOD, KW_DIV;
precedence left BITAND, BITOR, BITXOR, BITNOT;
+precedence left UNARYSIGN;
precedence left FACTORIAL;
precedence left KW_ORDER, KW_BY, KW_LIMIT;
precedence left LPAREN, RPAREN;
@@ -2608,8 +2610,10 @@ sign_chain_expr ::=
new NumericLiteral(BigDecimal.valueOf(-1)), e);
}
:}
+ %prec UNARYSIGN
| ADD expr:e
{: RESULT = e; :}
+ %prec UNARYSIGN
;
expr ::=
[3/6] incubator-impala git commit: IMPALA-5005: Don't allow server to
send SASL COMPLETE msg out of order
Posted by he...@apache.org.
IMPALA-5005: Don't allow server to send SASL COMPLETE msg out of order
Change-Id: I0c0d931d5d6ef3cbf50e2c36619cab2e0c72b629
Reviewed-on: http://gerrit.cloudera.org:8080/6190
Reviewed-by: Jim Apple <jb...@apache.org>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/c8480747
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/c8480747
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/c8480747
Branch: refs/heads/master
Commit: c84807479e129e97c84d60e5028c721bcf52f3d2
Parents: 93eb8cc
Author: Henry Robinson <he...@cloudera.com>
Authored: Mon Feb 27 16:47:48 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Mar 1 04:01:30 2017 +0000
----------------------------------------------------------------------
be/src/transport/TSaslTransport.cpp | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c8480747/be/src/transport/TSaslTransport.cpp
----------------------------------------------------------------------
diff --git a/be/src/transport/TSaslTransport.cpp b/be/src/transport/TSaslTransport.cpp
index 04f58cd..a64861b 100644
--- a/be/src/transport/TSaslTransport.cpp
+++ b/be/src/transport/TSaslTransport.cpp
@@ -98,7 +98,13 @@ namespace apache { namespace thrift { namespace transport {
while (!sasl_->isComplete()) {
uint8_t* message = receiveSaslMessage(&status, &resLength);
if (status == TSASL_COMPLETE) {
- if (isClient_) break; // handshake complete
+ if (isClient_) {
+ if (!sasl_->isComplete()) {
+ // Server sent COMPLETE out of order.
+ throw TTransportException("Received COMPLETE but no handshake occurred");
+ }
+ break; // handshake complete
+ }
} else if (status != TSASL_OK) {
stringstream ss;
ss << "Expected COMPLETE or OK, got " << status;
[2/6] incubator-impala git commit: Add a build flag for the undefined
behavior sanitizer, aka "ubsan".
Posted by he...@apache.org.
Add a build flag for the undefined behavior sanitizer, aka "ubsan".
Ubsan checks for undefined behavior according to the C++
standard. Some of this behavior has been known to be exploited by
optimizing compilers to produce bizarre results, like taking both
branches of a conditional.
This patch only adds build options; fixing the errors ubsan finds, as
well as adding any tests that a build is free from ubsan errors, are
not covered in this patch.
Change-Id: I03044c657ac171daa0648f833bbbeed7bdde49cb
Reviewed-on: http://gerrit.cloudera.org:8080/6186
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/93eb8ccb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/93eb8ccb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/93eb8ccb
Branch: refs/heads/master
Commit: 93eb8ccbedda060af01dccbb452c28d8f2072115
Parents: 1417d76
Author: Jim Apple <jb...@apache.org>
Authored: Tue Feb 28 12:01:13 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Mar 1 03:09:17 2017 +0000
----------------------------------------------------------------------
CMakeLists.txt | 3 ++-
be/CMakeLists.txt | 21 +++++++++++++++++++--
bin/make_impala.sh | 3 ++-
bin/run-backend-tests.sh | 3 +++
bin/start-impalad.sh | 2 ++
buildall.sh | 8 ++++++++
6 files changed, 36 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 80256cc..207fac4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -201,7 +201,8 @@ find_package(LlvmBinaries REQUIRED)
# Find LLVM libraries to link against.
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG"
OR "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER"
- OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY")
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY"
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
# Use the LLVM libaries with assertions for debug builds.
set(LLVM_ROOT ${LLVM_DEBUG_ROOT})
endif()
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/be/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 3697ceb..1e094a0 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -87,6 +87,19 @@ SET(CXX_FLAGS_RELEASE "${CXX_GCC_FLAGS} -O3 -DNDEBUG -gdwarf-2")
SET(CXX_FLAGS_ADDRESS_SANITIZER
"${CXX_CLANG_FLAGS} -O1 -g -fsanitize=address -fno-omit-frame-pointer -DADDRESS_SANITIZER")
+# Set the flags to the undefined behavior sanitizer, also known as "ubsan"
+# Turn on sanitizer and debug symbols to get stack traces:
+SET(CXX_FLAGS_UBSAN "${CXX_CLANG_FLAGS} -ggdb3 -fno-omit-frame-pointer -fsanitize=undefined")
+# Add flags to enable symbol resolution in the stack traces:
+SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -rtlib=compiler-rt -lgcc_s")
+# Ignore a number of noisy errors with too many false positives:
+SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -fno-sanitize=alignment,function,vptr,float-divide-by-zero,float-cast-overflow")
+# Don't enforce wrapped signed integer arithmetic so that the sanitizer actually sees
+# undefined wrapping:
+SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -fno-wrapv")
+# To ease debugging, turn off all optimizations:
+SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -O0")
+
SET(CXX_FLAGS_TIDY "${CXX_CLANG_FLAGS}")
# Catching unused variables requires an optimization level greater than 0
SET(CXX_FLAGS_TIDY "${CXX_FLAGS_TIDY} -O1")
@@ -111,6 +124,8 @@ elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER")
SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_ADDRESS_SANITIZER}")
elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "TIDY")
SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_TIDY}")
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
+ SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_UBSAN}")
else()
message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
endif()
@@ -133,7 +148,8 @@ if (CCACHE AND NOT DEFINED ENV{DISABLE_CCACHE})
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
if ("${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER"
- OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY")
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY"
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
# Need to set CCACHE_CPP so that ccache calls clang with the original source file for
# both preprocessing and compilation. Otherwise, ccache will use clang to preprocess
# the file and then call clang with the preprocessed output if not cached. However,
@@ -247,7 +263,8 @@ add_definitions(-fPIC)
# set compile output directory
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR
- "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER")
+ "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER" OR
+ "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/debug/")
else()
set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/release/")
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/bin/make_impala.sh
----------------------------------------------------------------------
diff --git a/bin/make_impala.sh b/bin/make_impala.sh
index 70d088f..8d164c2 100755
--- a/bin/make_impala.sh
+++ b/bin/make_impala.sh
@@ -148,7 +148,8 @@ then
if [[ ! -z $IMPALA_TOOLCHAIN ]]; then
if [[ ("$TARGET_BUILD_TYPE" == "ADDRESS_SANITIZER") \
- || ("$TARGET_BUILD_TYPE" == "TIDY") ]]
+ || ("$TARGET_BUILD_TYPE" == "TIDY") \
+ || ("$TARGET_BUILD_TYPE" == "UBSAN") ]]
then
CMAKE_ARGS+=(-DCMAKE_TOOLCHAIN_FILE=$IMPALA_HOME/cmake_modules/clang_toolchain.cmake)
else
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/bin/run-backend-tests.sh
----------------------------------------------------------------------
diff --git a/bin/run-backend-tests.sh b/bin/run-backend-tests.sh
index 758c1bc..f426542 100755
--- a/bin/run-backend-tests.sh
+++ b/bin/run-backend-tests.sh
@@ -34,6 +34,9 @@ fi
cd ${IMPALA_BE_DIR}
. ${IMPALA_HOME}/bin/set-classpath.sh
+cd ..
export CTEST_OUTPUT_ON_FAILURE=1
+export UBSAN_OPTIONS="print_stacktrace=1"
+export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
make test ARGS="${BE_TEST_ARGS}"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/bin/start-impalad.sh
----------------------------------------------------------------------
diff --git a/bin/start-impalad.sh b/bin/start-impalad.sh
index e7f2442..a731804 100755
--- a/bin/start-impalad.sh
+++ b/bin/start-impalad.sh
@@ -103,4 +103,6 @@ if ${CLUSTER_DIR}/admin is_kerberized; then
fi
. ${IMPALA_HOME}/bin/set-classpath.sh
+export UBSAN_OPTIONS='print_stacktrace=1'
+export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
exec ${TOOL_PREFIX} ${IMPALA_CMD} ${IMPALAD_ARGS}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/buildall.sh
----------------------------------------------------------------------
diff --git a/buildall.sh b/buildall.sh
index 9c4385c..bcc6ae3 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -59,6 +59,7 @@ CODE_COVERAGE=0
BUILD_ASAN=0
BUILD_FE_ONLY=0
BUILD_TIDY=0
+BUILD_UBSAN=0
# Export MAKE_CMD so it is visible in scripts that invoke make, e.g. copy-udfs-udas.sh
export MAKE_CMD=make
LZO_CMAKE_ARGS=
@@ -114,6 +115,9 @@ do
-tidy)
BUILD_TIDY=1
;;
+ -ubsan)
+ BUILD_UBSAN=1
+ ;;
-testpairwise)
EXPLORATION_STRATEGY=pairwise
;;
@@ -182,6 +186,7 @@ do
echo "[-codecoverage] : Build with code coverage [Default: False]"
echo "[-asan] : Address sanitizer build [Default: False]"
echo "[-tidy] : clang-tidy build [Default: False]"
+ echo "[-ubsan] : Undefined behavior build [Default: False]"
echo "[-skiptests] : Skips execution of all tests"
echo "[-notests] : Skips building and execution of all tests"
echo "[-start_minicluster] : Start test cluster including Impala and all"\
@@ -260,6 +265,9 @@ fi
if [[ ${BUILD_TIDY} -eq 1 ]]; then
CMAKE_BUILD_TYPE=TIDY
fi
+if [[ ${BUILD_UBSAN} -eq 1 ]]; then
+ CMAKE_BUILD_TYPE=UBSAN
+fi
MAKE_IMPALA_ARGS+=" -build_type=${CMAKE_BUILD_TYPE}"