You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/06/08 00:30:04 UTC
[1/2] incubator-impala git commit: IMPALA-3646: Handle corrupt RLE
literal or repeat counts of 0.
Repository: incubator-impala
Updated Branches:
refs/heads/master 2fb14d1b8 -> 025fd3bd7
IMPALA-3646: Handle corrupt RLE literal or repeat counts of 0.
Adds handling and testing for a specific Parquet data corruption
scenario with plain dictionary encoded values.
The problematic scenario is when the repeat or literal count of
the RLE-encoded dictionary indexes is decoded as 0 - an invalid value.
There are several other cases of data corruption that are not yet
handled gracefully. This patch only handles one specific case.
Change-Id: Ibf406c82cdded37966f09c81e4cc1446d2b60d63
Reviewed-on: http://gerrit.cloudera.org:8080/3299
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Alex Behm <al...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/025fd3bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/025fd3bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/025fd3bd
Branch: refs/heads/master
Commit: 025fd3bd7f3b7f4bc3dcb6b21e3598124b19b577
Parents: 86ff18e
Author: Alex Behm <al...@cloudera.com>
Authored: Thu Jun 2 00:03:20 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Tue Jun 7 17:29:59 2016 -0700
----------------------------------------------------------------------
be/src/exec/hdfs-parquet-scanner.cc | 5 ++--
be/src/util/rle-encoding.h | 3 ++-
be/src/util/rle-test.cc | 23 ++++++++++++++++++
testdata/data/README | 16 +++++++++++++
testdata/data/bad_rle_literal_count.parquet | Bin 0 -> 255 bytes
testdata/data/bad_rle_repeat_count.parquet | Bin 0 -> 234 bytes
.../parquet-corrupt-rle-counts-abort.test | 8 +++++++
.../QueryTest/parquet-corrupt-rle-counts.test | 9 +++++++
tests/query_test/test_scanners.py | 24 +++++++++++++++++++
9 files changed, 85 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/025fd3bd/be/src/exec/hdfs-parquet-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner.cc b/be/src/exec/hdfs-parquet-scanner.cc
index 4bb7bac..1713af6 100644
--- a/be/src/exec/hdfs-parquet-scanner.cc
+++ b/be/src/exec/hdfs-parquet-scanner.cc
@@ -866,7 +866,7 @@ class HdfsParquetScanner::ScalarColumnReader :
if (def_level >= max_def_level()) {
bool continue_execution =
ReadSlot<IS_DICT_ENCODED>(tuple->GetSlot(tuple_offset_), pool);
- if (UNLIKELY(!continue_execution)) break;
+ if (UNLIKELY(!continue_execution)) return false;
} else {
tuple->SetNull(null_indicator_offset_);
}
@@ -1622,7 +1622,8 @@ bool HdfsParquetScanner::LevelDecoder::FillCache(int batch_size,
}
literal_count_ -= num_literals_to_set;
- if (num_values == batch_size || !NextCounts<int16_t>()) break;
+ if (num_values == batch_size) break;
+ if (UNLIKELY(!NextCounts<int16_t>())) return false;
if (repeat_count_ > 0 && current_value_ > max_level_) return false;
}
} else {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/025fd3bd/be/src/util/rle-encoding.h
----------------------------------------------------------------------
diff --git a/be/src/util/rle-encoding.h b/be/src/util/rle-encoding.h
index c6686e3..030767d 100644
--- a/be/src/util/rle-encoding.h
+++ b/be/src/util/rle-encoding.h
@@ -278,11 +278,12 @@ bool RleDecoder::NextCounts() {
bool is_literal = indicator_value & 1;
if (is_literal) {
literal_count_ = (indicator_value >> 1) * 8;
+ if (UNLIKELY(literal_count_ == 0)) return false;
} else {
repeat_count_ = indicator_value >> 1;
bool result = bit_reader_.GetAligned<T>(
BitUtil::Ceil(bit_width_, 8), reinterpret_cast<T*>(¤t_value_));
- DCHECK(result);
+ if (UNLIKELY(!result || repeat_count_ == 0)) return false;
}
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/025fd3bd/be/src/util/rle-test.cc
----------------------------------------------------------------------
diff --git a/be/src/util/rle-test.cc b/be/src/util/rle-test.cc
index 8073068..691dd45 100644
--- a/be/src/util/rle-test.cc
+++ b/be/src/util/rle-test.cc
@@ -419,6 +419,29 @@ TEST(BitRle, Overflow) {
}
}
+// Tests handling of a specific data corruption scenario where
+// the literal or repeat count is decoded as 0 (which is invalid).
+TEST(Rle, ZeroLiteralOrRepeatCount) {
+ const int len = 1024;
+ uint8_t buffer[len];
+ RleDecoder decoder(buffer, len, 0);
+ uint64_t val;
+
+ // Test the RLE repeated values path.
+ memset(buffer, 0, len);
+ for (int i = 0; i < 10; ++i) {
+ bool result = decoder.Get(&val);
+ EXPECT_FALSE(result);
+ }
+
+ // Test the RLE literal values path
+ memset(buffer, 1, len);
+ for (int i = 0; i < 10; ++i) {
+ bool result = decoder.Get(&val);
+ EXPECT_FALSE(result);
+ }
+}
+
}
int main(int argc, char **argv) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/025fd3bd/testdata/data/README
----------------------------------------------------------------------
diff --git a/testdata/data/README b/testdata/data/README
index f8cabda..53993c8 100644
--- a/testdata/data/README
+++ b/testdata/data/README
@@ -5,6 +5,22 @@ Contains 3 single-column rows:
"is"
"fun"
+bad_rle_literal_count.parquet:
+Generated by hacking Impala's Parquet writer.
+Contains a single bigint column 'c' with the values 1, 3, 7 stored
+in a single data chunk as dictionary plain. The RLE encoded dictionary
+indexes are all literals (and not repeated), but the literal count
+is incorrectly 0 in the file to test that such data corruption is
+proprly handled.
+
+bad_rle_repeat_count.parquet:
+Generated by hacking Impala's Parquet writer.
+Contains a single bigint column 'c' with the value 7 repeated 7 times
+stored in a single data chunk as dictionary plain. The RLE encoded dictionary
+indexes are a single repeated run (and not literals), but the repeat count
+is incorrectly 0 in the file to test that such data corruption is proprly
+handled.
+
repeated_values.parquet:
Generated with parquet-mr 1.2.5
Contains 3 single-column rows:
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/025fd3bd/testdata/data/bad_rle_literal_count.parquet
----------------------------------------------------------------------
diff --git a/testdata/data/bad_rle_literal_count.parquet b/testdata/data/bad_rle_literal_count.parquet
new file mode 100644
index 0000000..33d21ac
Binary files /dev/null and b/testdata/data/bad_rle_literal_count.parquet differ
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/025fd3bd/testdata/data/bad_rle_repeat_count.parquet
----------------------------------------------------------------------
diff --git a/testdata/data/bad_rle_repeat_count.parquet b/testdata/data/bad_rle_repeat_count.parquet
new file mode 100644
index 0000000..ad468ac
Binary files /dev/null and b/testdata/data/bad_rle_repeat_count.parquet differ
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/025fd3bd/testdata/workloads/functional-query/queries/QueryTest/parquet-corrupt-rle-counts-abort.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/parquet-corrupt-rle-counts-abort.test b/testdata/workloads/functional-query/queries/QueryTest/parquet-corrupt-rle-counts-abort.test
new file mode 100644
index 0000000..e0789ab
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/parquet-corrupt-rle-counts-abort.test
@@ -0,0 +1,8 @@
+====
+---- QUERY
+select * from bad_rle_counts
+---- TYPES
+bigint
+---- CATCH
+Failed to decode dictionary-encoded value.
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/025fd3bd/testdata/workloads/functional-query/queries/QueryTest/parquet-corrupt-rle-counts.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/parquet-corrupt-rle-counts.test b/testdata/workloads/functional-query/queries/QueryTest/parquet-corrupt-rle-counts.test
new file mode 100644
index 0000000..c086278
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/parquet-corrupt-rle-counts.test
@@ -0,0 +1,9 @@
+====
+---- QUERY
+select * from bad_rle_counts
+---- TYPES
+bigint
+---- RESULTS
+---- ERRORS
+Failed to decode dictionary-encoded value. file: hdfs://regex:.$
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/025fd3bd/tests/query_test/test_scanners.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 216d3df..035b64b 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -224,6 +224,30 @@ class TestParquet(ImpalaTestSuite):
vector.get_value('exec_option')['abort_on_error'] = 1
self.run_test_case('QueryTest/parquet-abort-on-error', vector)
+ def test_corrupt_rle_counts(self, vector, unique_database):
+ """IMPALA-3646: Tests that a certain type of file corruption for plain
+ dictionary encoded values is gracefully handled. Cases tested:
+ - incorrect literal count of 0 for the RLE encoded dictionary indexes
+ - incorrect repeat count of 0 for the RLE encoded dictionary indexes
+ """
+ # Create test table and copy the corrupt files into it.
+ self.client.execute(
+ "create table %s.bad_rle_counts (c bigint) stored as parquet" % unique_database)
+ bad_rle_counts_tbl_loc =\
+ get_fs_path("/test-warehouse/%s.db/%s" % (unique_database, "bad_rle_counts"))
+ check_call(['hdfs', 'dfs', '-copyFromLocal',
+ os.environ['IMPALA_HOME'] + "/testdata/data/bad_rle_literal_count.parquet",
+ bad_rle_counts_tbl_loc])
+ check_call(['hdfs', 'dfs', '-copyFromLocal',
+ os.environ['IMPALA_HOME'] + "/testdata/data/bad_rle_repeat_count.parquet",
+ bad_rle_counts_tbl_loc])
+ # Querying the corrupted files should not DCHECK or crash.
+ vector.get_value('exec_option')['abort_on_error'] = 0
+ self.run_test_case('QueryTest/parquet-corrupt-rle-counts', vector, unique_database)
+ vector.get_value('exec_option')['abort_on_error'] = 1
+ self.run_test_case('QueryTest/parquet-corrupt-rle-counts-abort',
+ vector, unique_database)
+
@SkipIfS3.hdfs_block_size
@SkipIfIsilon.hdfs_block_size
@SkipIfLocal.multiple_impalad
[2/2] incubator-impala git commit: IMPALA-3223: Removal of
non-toolchain builds.
Posted by ta...@apache.org.
IMPALA-3223: Removal of non-toolchain builds.
This change removes the option to build without specifying
the environment variable $IMPALA_TOOLCHAIN. By default, if
it's not set, sourcing impala-config.sh will set it to
$IMPALA_HOME/toolchain. A user can override it by setting
$IMPALA_TOOLCHAIN to his/her own toolchain directory. The
user can also set $SKIP_TOOLCHAIN_BOOTSTRAP to true to
avoid running the toolchain bootstrapping script (e.g. a
particular component in toolchain is at a version not
checked into S3).
$IMPALA_TOOLCHAIN holds some third party binaries which
Impala relies on. They can be compiled from source in the
native toolchain which is public. This commit also removes
build_thirdparty.sh as it's no longer used.
By default, Impala will be built with the compiler in
$IMPALA_TOOLCHAIN but this option can be overridden by
setting environment variable $USE_SYSTEM_GCC to 1.
Change-Id: I42b60e99fb9caf1294be7ab242856ca3b9a5ab73
Reviewed-on: http://gerrit.cloudera.org:8080/3259
Reviewed-by: Michael Ho <kw...@cloudera.com>
Tested-by: Michael Ho <kw...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/86ff18ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/86ff18ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/86ff18ee
Branch: refs/heads/master
Commit: 86ff18eee9cf3babe0d596050088bf75b5287467
Parents: 2fb14d1
Author: Michael Ho <kw...@cloudera.com>
Authored: Fri May 27 19:19:36 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Tue Jun 7 17:29:59 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 148 +++++++++-----------
be/CMakeLists.txt | 38 ++---
bin/build_thirdparty.sh | 241 --------------------------------
bin/impala-config.sh | 109 ++++++---------
buildall.sh | 14 +-
cmake_modules/FindPICLibs.cmake | 19 ---
cmake_modules/asan_toolchain.cmake | 2 -
cmake_modules/toolchain.cmake | 2 -
8 files changed, 116 insertions(+), 457 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index dc0ec57..0c09ccd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -14,28 +14,19 @@
cmake_minimum_required(VERSION 2.6)
-if (NOT $ENV{DISABLE_IMPALA_TOOLCHAIN})
- # Enable toolchain variable if the environment is setup
- set(IMPALA_TOOLCHAIN ON)
- message(STATUS "Toolchain build.")
- if ("$ENV{SKIP_TOOLCHAIN_BOOTSTRAP}" STREQUAL "true")
- message(STATUS "SKIP_TOOLCHAIN_BOOTSTRAP is true, skipping toolchain bootstrap.")
+if ("$ENV{SKIP_TOOLCHAIN_BOOTSTRAP}" STREQUAL "true")
+ message(STATUS "SKIP_TOOLCHAIN_BOOTSTRAP is true, skipping toolchain bootstrap.")
+else()
+ # Download any missing toolchain dependencies. If this fails, fail the build.
+ set(BOOTSTRAP_CMD "$ENV{IMPALA_HOME}/bin/bootstrap_toolchain.py")
+ # Download and unpack the dependencies
+ message(STATUS "Downloading and extracting dependencies.")
+ execute_process(COMMAND ${BOOTSTRAP_CMD} RESULT_VARIABLE BOOTSTRAP_RESULT)
+ if (${BOOTSTRAP_RESULT} EQUAL 0)
+ message(STATUS "Toolchain bootstrap complete.")
else()
- # Download any missing toolchain dependencies. If this fails, fail the build.
- set(BOOTSTRAP_CMD "$ENV{IMPALA_HOME}/bin/bootstrap_toolchain.py")
- # Download and unpack the dependencies
- message(STATUS "Downloading and extracting dependencies.")
- execute_process(COMMAND ${BOOTSTRAP_CMD} RESULT_VARIABLE BOOTSTRAP_RESULT)
- if (${BOOTSTRAP_RESULT} EQUAL 0)
- message(STATUS "Toolchain bootstrap complete.")
- else()
- message(FATAL_ERROR "Toolchain bootstrap failed.")
- endif()
+ message(FATAL_ERROR "Toolchain bootstrap failed.")
endif()
-else()
- set(IMPALA_TOOLCHAIN OFF)
- message(STATUS "Impala toolchain was explicitly disabled using DISABLE_IMPALA_TOOLCHAIN.")
- message(STATUS "Assuming system search path for dependencies.")
endif()
# Explicitly define project() to allow modifying the compiler before the project is
@@ -84,63 +75,51 @@ function(set_dep_root NAME)
set(${NAME}_ROOT $ENV{IMPALA_TOOLCHAIN}/${NAME_LOWER}-$ENV{${VAL_NAME}} PARENT_SCOPE)
endfunction()
-set(Boost_USE_MULTITHREADED ON)
-
-if (IMPALA_TOOLCHAIN)
- message(STATUS "Found Impala Toolchain, enabling toolchain boost")
- set(ENV{BOOST_ROOT} "$ENV{IMPALA_TOOLCHAIN}/boost-$ENV{IMPALA_BOOST_VERSION}")
- set(Boost_USE_STATIC_LIBS NOT ${BUILD_SHARED_LIBS})
- set(Boost_USE_STATIC_RUNTIME ON)
-
- # Newer versions of boost (including the version in toolchain) don't build separate
- # multithreaded versions (they always are). Make sure to pick those up.
- set(Boost_USE_MULTITHREADED OFF)
-
- # Define root path for all dependencies, this is in the form of
- # set_dep_root(PACKAGE) ->
- # PACKAGE_ROOT set to $ENV{IMPALA_TOOLCHAIN}/PACKAGE-$ENV{IMPALA_PACKAGE_VERSION}
- set_dep_root(GCC)
- set_dep_root(AVRO)
- set_dep_root(BREAKPAD)
- set_dep_root(BZIP2)
- set_dep_root(GFLAGS)
- set_dep_root(GLOG)
- set_dep_root(GPERFTOOLS)
- set_dep_root(GTEST)
- set_dep_root(LLVM)
- set(LLVM_DEBUG_ROOT $ENV{IMPALA_TOOLCHAIN}/llvm-$ENV{IMPALA_LLVM_DEBUG_VERSION})
- set_dep_root(LZ4)
- set_dep_root(OPENLDAP)
- set_dep_root(RE2)
- set_dep_root(RAPIDJSON)
- set_dep_root(SNAPPY)
- set_dep_root(THRIFT)
- set_dep_root(ZLIB)
- if (APPLE)
- set_dep_root(OPENSSL)
- endif()
-endif ()
+# Define root path for all dependencies, this is in the form of
+# set_dep_root(PACKAGE) ->
+# PACKAGE_ROOT set to $ENV{IMPALA_TOOLCHAIN}/PACKAGE-$ENV{IMPALA_PACKAGE_VERSION}
+set_dep_root(AVRO)
+set_dep_root(BOOST)
+set_dep_root(BREAKPAD)
+set_dep_root(BZIP2)
+set_dep_root(GCC)
+set_dep_root(GFLAGS)
+set_dep_root(GLOG)
+set_dep_root(GPERFTOOLS)
+set_dep_root(GTEST)
+set_dep_root(LLVM)
+set(LLVM_DEBUG_ROOT $ENV{IMPALA_TOOLCHAIN}/llvm-$ENV{IMPALA_LLVM_DEBUG_VERSION})
+set_dep_root(LZ4)
+set_dep_root(OPENLDAP)
+set_dep_root(RE2)
+set_dep_root(RAPIDJSON)
+set_dep_root(SNAPPY)
+set_dep_root(THRIFT)
+set_dep_root(ZLIB)
+if (APPLE)
+ set_dep_root(OPENSSL)
+endif()
+
+set(Boost_USE_STATIC_LIBS NOT ${BUILD_SHARED_LIBS})
+set(Boost_USE_STATIC_RUNTIME ON)
+
+# Newer versions of boost (including the version in toolchain) don't build separate
+# multithreaded versions (they always are). Make sure to pick those up.
+# TODO: understand the consequence of leaving this ON (the default value).
+set(Boost_USE_MULTITHREADED OFF)
-# find libraries that will be statically linked if they were compiled with -fPIC
-find_package(PICLibs REQUIRED)
-
-# find boost headers and libs
-IF (DEFINED ENV{BOOST_ROOT})
- # The casing and underscoring expected for these properties varies between
- # versions of CMake. Multiple inconsistent versions may be present here
- # intentionally to provide what a wide range of versions expects.
- set(Boost_NO_SYSTEM_PATHS true)
- set(BOOST_ROOT $ENV{BOOST_ROOT})
- set(BOOST_LIBRARYDIR $ENV{BOOST_ROOT}/lib)
- set(BOOST_INCLUDEDIR $ENV{BOOST_ROOT}/include)
- set(Boost_INCLUDE_DIR ${BOOST_INCLUDEDIR})
-ENDIF (DEFINED ENV{BOOST_ROOT})
+# The casing and underscoring expected for these properties varies between
+# versions of CMake. Multiple inconsistent versions may be present here
+# intentionally to provide what a wide range of versions expects.
+set(Boost_NO_SYSTEM_PATHS true)
+set(BOOST_LIBRARYDIR ${BOOST_ROOT}/lib)
+set(BOOST_INCLUDEDIR ${BOOST_ROOT}/include)
+set(Boost_INCLUDE_DIR ${BOOST_INCLUDEDIR})
if (CMAKE_DEBUG)
set(Boost_DEBUG TRUE)
endif()
-
find_package(Boost $ENV{IMPALA_BOOST_VERSION} REQUIRED
COMPONENTS thread regex system filesystem date_time)
include_directories(${Boost_INCLUDE_DIRS})
@@ -160,20 +139,17 @@ find_package(Zlib REQUIRED)
include_directories(${ZLIB_INCLUDE_DIR})
if(NOT BUILD_SHARED_LIBS)
- # It might be the case that bzip2 and zlib are not available as static libs in the non
- # toolchain case
- # TODO remove this when moving to toolchain only
+ # Panic if we cannot find the static libraries as they are supposed to be
+ # in the toolchain.
if (NOT BZIP2_STATIC_FOUND)
- set(LIBBZ2 ${BZIP2_LIBRARIES})
- else()
- set(LIBBZ2 BZIP2_STATIC)
+ message(FATAL_ERROR "bzip2 static library libbz2.a not found in $ENV{IMPALA_TOOLCHAIN}")
endif()
-
if (NOT ZLIB_STATIC_FOUND)
- set(LIBZ ${ZLIB_LIBRARIES})
- else()
- set(LIBZ ZLIB_STATIC)
+ message(FATAL_ERROR "zlib static library libz.a not found in $ENV{IMPALA_TOOLCHAIN}")
endif()
+
+ set(LIBBZ2 BZIP2_STATIC)
+ set(LIBZ ZLIB_STATIC)
else()
set(LIBBZ2 ${BZIP2_LIBRARIES})
set(LIBZ ${ZLIB_LIBRARIES})
@@ -226,8 +202,8 @@ set(LLVM_BINARIES_ROOT ${LLVM_ROOT})
find_package(LlvmBinaries REQUIRED)
# Find LLVM libraries to link against.
-if (IMPALA_TOOLCHAIN AND ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG"
- OR "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER"))
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG"
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER")
# Use the LLVM libaries with assertions for debug builds.
set(LLVM_ROOT ${LLVM_DEBUG_ROOT})
endif()
@@ -253,6 +229,12 @@ set(LDAP_LIBRARY ldapstatic)
set(LBER_LIBRARY lberstatic)
message(STATUS "LDAP: ${LDAP_INCLUDE_DIR}")
+# The environment variable $THRIFT_HOME is set in impala-config.sh
+# Make sure it's consistent with $THRIFT_ROOT.
+if (NOT ($ENV{THRIFT_HOME} STREQUAL ${THRIFT_ROOT}))
+ message(FATAL_ERROR "THRIFT_ROOT (${THRIFT_ROOT}) differs from environment "
+ "variable THRIFT_HOME ($ENV{THRIFT_HOME}).")
+endif()
# find thrift headers and libs
find_package(Thrift REQUIRED)
include_directories(${THRIFT_INCLUDE_DIR})
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/be/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index ed4a8c4..5d3e246 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -43,9 +43,7 @@ SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated -Wno-vla")
SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG")
IF($ENV{USE_GOLD_LINKER} STREQUAL "true")
SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wl,-fuse-ld=gold")
- if (IMPALA_TOOLCHAIN)
- SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -B $ENV{IMPALA_TOOLCHAIN}/binutils-$ENV{IMPALA_BINUTILS_VERSION}/bin/")
- endif()
+ SET(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -B $ENV{IMPALA_TOOLCHAIN}/binutils-$ENV{IMPALA_BINUTILS_VERSION}/bin/")
ENDIF()
# On Apple we build with clang and need libstdc++ instead of libc++
@@ -78,11 +76,9 @@ SET(CXX_GCC_FLAGS "-g -Wno-unused-local-typedefs")
#
# Debug information is stored as dwarf2 to be as compatible as possible
SET(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -ggdb -gdwarf-2")
-if (IMPALA_TOOLCHAIN)
- # -Werror: compile warnings should be errors when using the toolchain compiler.
- # Only enable for debug builds because this is what we test in pre-commit tests.
- SET(CXX_FLAGS_DEBUG "${CXX_FLAGS_DEBUG} -Werror")
-endif()
+# -Werror: compile warnings should be errors when using the toolchain compiler.
+# Only enable for debug builds because this is what we test in pre-commit tests.
+SET(CXX_FLAGS_DEBUG "${CXX_FLAGS_DEBUG} -Werror")
SET(CXX_FLAGS_RELEASE "${CXX_GCC_FLAGS} -O3 -DNDEBUG -gdwarf-2")
SET(CXX_FLAGS_ADDRESS_SANITIZER
"${CXX_CLANG_FLAGS} -O1 -g -fsanitize=address -fno-omit-frame-pointer -DADDRESS_SANITIZER")
@@ -93,9 +89,6 @@ if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
SET(CMAKE_CXX_FLAGS ${CXX_FLAGS_RELEASE})
elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER")
- if (NOT IMPALA_TOOLCHAIN)
- SET(CMAKE_CXX_COMPILER ${LLVM_CLANG_EXECUTABLE})
- endif()
SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_ADDRESS_SANITIZER}")
else()
message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
@@ -145,10 +138,9 @@ set(CLANG_IR_CXX_FLAGS "-emit-llvm" "-c" "-std=c++14" "-DIR_COMPILE" "-DNDEBUG"
"-DHAVE_INTTYPES_H" "-DHAVE_NETINET_IN_H" "-DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG"
"-DBOOST_NO_EXCEPTIONS" "-fcolor-diagnostics" "-Wno-deprecated"
"-Wno-return-type-c-linkage" "-O1")
-if (IMPALA_TOOLCHAIN)
- # -Werror: compile warnings should be errors when using the toolchain compiler.
- SET(CLANG_IR_CXX_FLAGS "${CLANG_IR_CXX_FLAGS}" "-Werror")
-endif()
+# -Werror: compile warnings should be errors when using the toolchain compiler.
+set(CLANG_IR_CXX_FLAGS "${CLANG_IR_CXX_FLAGS}" "-Werror")
+
IF($ENV{ENABLE_IMPALA_IR_DEBUG_INFO} STREQUAL "true")
# -g: emit debug symbols in IR. These increase IR size and memory overhead of LLVM, but
# are useful for debugging codegened code and interpreting codegen disassembly
@@ -160,7 +152,6 @@ endif()
# -inline: inline with low threshold to get rid of trivial accessor functions.
set(LLVM_OPT_IR_FLAGS "-inline" "-inlinehint-threshold=10" "-inline-threshold=10")
-
# setup doc generation with Doxygen
find_package(Doxygen)
if (DOXYGEN_FOUND)
@@ -203,11 +194,9 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/generated-sources)
set(CLANG_INCLUDE_FLAGS)
-if (IMPALA_TOOLCHAIN)
- # Ensure that clang uses the gcc toolchain headers.
- set(CLANG_BASE_FLAGS --gcc-toolchain=${GCC_ROOT})
- set(CLANG_INCLUDE_FLAGS ${CLANG_BASE_FLAGS})
-endif()
+# Ensure that clang uses the gcc toolchain headers.
+set(CLANG_BASE_FLAGS --gcc-toolchain=${GCC_ROOT})
+set(CLANG_INCLUDE_FLAGS ${CLANG_BASE_FLAGS})
set(CLANG_INCLUDE_FLAGS
${CLANG_INCLUDE_FLAGS}
@@ -355,13 +344,6 @@ endif()
set(IMPALA_LINK_LIBS_DYNAMIC_TARGETS ${IMPALA_LINK_LIBS_NO_TCMALLOC})
list(REMOVE_ITEM IMPALA_LINK_LIBS_DYNAMIC_TARGETS HDFS_STATIC)
set(IMPALA_LINK_LIBS_DYNAMIC_TARGETS ${IMPALA_LINK_LIBS_DYNAMIC_TARGETS} ${HDFS_LIBRARIES})
-# Replace static libz bzib with dynamic
-if (NOT IMPALA_TOOLCHAIN AND NOT BUILD_SHARED_LIBS)
- list(REMOVE_ITEM IMPALA_LINK_LIBS_DYNAMIC_TARGETS BZIP2_STATIC)
- list(REMOVE_ITEM IMPALA_LINK_LIBS_DYNAMIC_TARGETS ZLIB_STATIC)
- set(IMPALA_LINK_LIBS_DYNAMIC_TARGETS
- ${IMPALA_LINK_LIBS_DYNAMIC_TARGETS} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES})
-endif()
# Link libs for test executables. Although not all tests need all libs,
# the build time for the tests is rather small and not worth the trouble.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/bin/build_thirdparty.sh
----------------------------------------------------------------------
diff --git a/bin/build_thirdparty.sh b/bin/build_thirdparty.sh
deleted file mode 100755
index ce1f323..0000000
--- a/bin/build_thirdparty.sh
+++ /dev/null
@@ -1,241 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2012 Cloudera Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# cleans and rebuilds thirdparty/. The Impala build environment must be set up
-# by bin/impala-config.sh before running this script.
-
-# Exit on non-true return value
-set -e
-# Exit on reference to uninitialized variable
-set -u
-
-# By default, git clean every library we build
-CLEAN_ACTION=1
-
-# By default, build every library
-BUILD_ALL=1
-
-# If BUILD_ALL -eq 1, these are ignored, otherwise only build those libraries with
-# BUILD_<lib> -eq 1
-BUILD_AVRO=0
-BUILD_THRIFT=0
-BUILD_GLOG=0
-BUILD_GFLAGS=0
-BUILD_GTEST=0
-BUILD_RE2=0
-BUILD_SASL=0
-BUILD_LDAP=0
-BUILD_SNAPPY=0
-BUILD_PPROF=0
-BUILD_CDH4EXTRAS=0
-BUILD_LZ4=0
-
-for ARG in $*
-do
- case "$ARG" in
- -noclean)
- CLEAN_ACTION=0
- ;;
- -avro)
- BUILD_ALL=0
- BUILD_AVRO=1
- ;;
- -glog)
- BUILD_ALL=0
- BUILD_GLOG=1
- ;;
- -thrift)
- BUILD_ALL=0
- BUILD_THRIFT=1
- ;;
- -gflags)
- BUILD_ALL=0
- BUILD_GFLAGS=1
- ;;
- -gtest)
- BUILD_ALL=0
- BUILD_GTEST=1
- ;;
- -re2)
- BUILD_ALL=0
- BUILD_RE2=1
- ;;
- -sasl)
- BUILD_ALL=0
- BUILD_SASL=1
- ;;
- -ldap)
- BUILD_ALL=0
- BUILD_LDAP=1
- ;;
- -snappy)
- BUILD_ALL=0
- BUILD_SNAPPY=1
- ;;
- -lz4)
- BUILD_ALL=0
- BUILD_LZ4=1
- ;;
- -pprof)
- BUILD_ALL=0
- BUILD_PPROF=1
- ;;
- -*)
- echo "Usage: build_thirdparty.sh [-noclean] \
-[-avro -glog -thrift -gflags -gtest -re2 -sasl -ldap -snappy -pprof]"
- exit 1
- esac
-done
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-. "$bin"/impala-config.sh
-USE_PIC_LIB_PATH=${PIC_LIB_PATH:-}
-
-function build_preamble() {
- echo
- echo
- echo "********************************************************************************"
- echo "Building $2 in $1 $([ $CLEAN_ACTION -eq 1 ] && echo '(clean)')"
- echo "********************************************************************************"
- cd $1
- if [ $CLEAN_ACTION -eq 1 ]; then
- # remove everything that is not checked in
- git clean -dfx
- fi
-}
-
-# Build Sasl
-if [ $BUILD_ALL -eq 1 ] || [ $BUILD_SASL -eq 1 ]; then
- build_preamble $IMPALA_HOME/thirdparty/cyrus-sasl-${IMPALA_CYRUS_SASL_VERSION} Sasl
-
- # Need to specify which libdb to use on certain OSes
- LIBDB_DIR=""
- if [[ -e "/usr/lib64/libdb4" && -e "/usr/include/libdb4" ]]; then
- LIBDB_DIR="--with-bdb-libdir=/usr/lib64/libdb4 --with-bdb-incdir=/usr/include/libdb4"
- fi
- # Disable everything except those protocols needed -- currently just Kerberos.
- # Sasl does not have a --with-pic configuration.
- CFLAGS="-fPIC -DPIC" CXXFLAGS="-fPIC -DPIC" ./configure \
- --disable-sql --disable-otp --disable-ldap --disable-digest --with-saslauthd=no \
- --prefix=$IMPALA_CYRUS_SASL_INSTALL_DIR --enable-static --enable-staticdlopen \
- $LIBDB_DIR
- # the first time you do a make it fails, build again.
- (make || make)
- make install
-fi
-
-set -e
-# build thrift
-if [ $BUILD_ALL -eq 1 ] || [ $BUILD_THRIFT -eq 1 ]; then
- cd ${THRIFT_SRC_DIR}
- build_preamble ${THRIFT_SRC_DIR} "Thrift"
- if [ -d "${PIC_LIB_PATH:-}" ]; then
- PIC_LIB_OPTIONS="--with-zlib=${PIC_LIB_PATH} "
- fi
- JAVA_PREFIX=${THRIFT_HOME}/java PY_PREFIX=${THRIFT_HOME}/python \
- ./configure --with-pic --prefix=${THRIFT_HOME} \
- --with-php=no --with-java=no --with-perl=no --with-erlang=no \
- --with-ruby=no --with-haskell=no --with-erlang=no --with-d=no \
- --with-go=no --with-qt4=no --with-libevent=no ${PIC_LIB_OPTIONS:-}
- make # Make with -j fails
- make install
- cd ${THRIFT_SRC_DIR}/contrib/fb303
- chmod 755 ./bootstrap.sh
- ./bootstrap.sh
- chmod 755 configure
- CPPFLAGS="-I${THRIFT_HOME}/include" PY_PREFIX=${THRIFT_HOME}/python ./configure \
- --with-java=no --with-php=no --prefix=${THRIFT_HOME} --with-thriftpath=${THRIFT_HOME}
- make
- make install
-fi
-
-# build gflags
-if [ $BUILD_ALL -eq 1 ] || [ $BUILD_GFLAGS -eq 1 ]; then
- build_preamble $IMPALA_HOME/thirdparty/gflags-${IMPALA_GFLAGS_VERSION} GFlags
- GFLAGS_INSTALL=`pwd`/third-party-install
- ./configure --with-pic --prefix=${GFLAGS_INSTALL}
- make -j${IMPALA_BUILD_THREADS:-4} install
-fi
-
-# Build pprof
-if [ $BUILD_ALL -eq 1 ] || [ $BUILD_PPROF -eq 1 ]; then
- build_preamble $IMPALA_HOME/thirdparty/gperftools-${IMPALA_GPERFTOOLS_VERSION} \
- GPerftools
- # TODO: google perf tools indicates this might be necessary on 64 bit systems.
- # we're not compiling the rest of our code to not omit frame pointers but it
- # still seems to generate useful profiling data.
- ./configure --enable-frame-pointers --with-pic
- make -j${IMPALA_BUILD_THREADS:-4}
-fi
-
-# Build glog
-if [ $BUILD_ALL -eq 1 ] || [ $BUILD_GLOG -eq 1 ]; then
- build_preamble $IMPALA_HOME/thirdparty/glog-${IMPALA_GLOG_VERSION} GLog
- ./configure --with-pic --with-gflags=${GFLAGS_INSTALL}
- # SLES's gcc45-c++ is required for sse2 support (default is 4.3), but crashes
- # when building logging_unittest-logging_unittest.o. Telling it to uses the
- # stabs format for debugging symbols instead of dwarf exercises a different
- # code path to work around this issue.
- cat > Makefile.gcc45sles_workaround <<EOF
-logging_unittest-logging_unittest.o : CXXFLAGS= -gstabs -O2
-EOF
- cat Makefile >> Makefile.gcc45sles_workaround
- mv Makefile.gcc45sles_workaround Makefile
- make -j${IMPALA_BUILD_THREADS:-4}
-fi
-
-# Build gtest
-if [ $BUILD_ALL -eq 1 ] || [ $BUILD_GTEST -eq 1 ]; then
- build_preamble $IMPALA_HOME/thirdparty/gtest-${IMPALA_GTEST_VERSION} GTest
- cmake .
- make -j${IMPALA_BUILD_THREADS:-4}
-fi
-
-# Build Snappy
-if [ $BUILD_ALL -eq 1 ] || [ $BUILD_SNAPPY -eq 1 ]; then
- build_preamble $IMPALA_HOME/thirdparty/snappy-${IMPALA_SNAPPY_VERSION} Snappy
- ./autogen.sh
- ./configure --with-pic --prefix=$IMPALA_HOME/thirdparty/snappy-${IMPALA_SNAPPY_VERSION}/build
- make install
-fi
-
-# Build Lz4
-if [ $BUILD_ALL -eq 1 ] || [ $BUILD_LZ4 -eq 1 ]; then
- build_preamble $IMPALA_HOME/thirdparty/lz4 Lz4
- cmake .
- make
-fi
-
-# Build re2
-if [ $BUILD_ALL -eq 1 ] || [ $BUILD_RE2 -eq 1 ]; then
- build_preamble $IMPALA_HOME/thirdparty/re2 RE2
- make -j${IMPALA_BUILD_THREADS:-4}
-fi
-
-# Build Ldap
-if [ $BUILD_ALL -eq 1 ] || [ $BUILD_LDAP -eq 1 ]; then
- build_preamble $IMPALA_HOME/thirdparty/openldap-${IMPALA_OPENLDAP_VERSION} Openldap
- ./configure --enable-slapd=no --prefix=`pwd`/impala_install --enable-static --with-pic
- make -j${IMPALA_BUILD_THREADS:-4}
- make -j${IMPALA_BUILD_THREADS:-4} depend
- make install
-fi
-
-# Build Avro
-if [ $BUILD_ALL -eq 1 ] || [ $BUILD_AVRO -eq 1 ]; then
- build_preamble $IMPALA_HOME/thirdparty/avro-c-${IMPALA_AVRO_VERSION} Avro
- cmake .
- make -j${IMPALA_BUILD_THREADS:-4}
-fi
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 89b7400..d6a0833 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -21,7 +21,7 @@
# this script because scripts outside this repository may need to be updated and that
# is not practical at this time.
export JAVA_HOME="${JAVA_HOME:-/usr/java/default}"
-if [ ! -d "$JAVA_HOME" ] ; then
+if [ ! -d "$JAVA_HOME" ]; then
echo "JAVA_HOME must be set to the location of your JDK!"
return 1
fi
@@ -39,8 +39,15 @@ if [ -z $IMPALA_HOME ]; then
fi
fi
-: ${DISABLE_IMPALA_TOOLCHAIN=0}
: ${IMPALA_TOOLCHAIN=$IMPALA_HOME/toolchain}
+if [ -z $IMPALA_TOOLCHAIN ]; then
+ echo "IMPALA_TOOLCHAIN must be specified. Please set it to a valid directory or"\
+ "leave it unset."
+ return 1
+fi
+
+# This flag is used in $IMPALA_HOME/cmake_modules/toolchain.cmake.
+# If it's 0, Impala will be built with the compiler in the toolchain directory.
: ${USE_SYSTEM_GCC=0}
# Gold is available on newer systems and a full build with static linking is ~1.5 mins
@@ -48,15 +55,14 @@ fi
: ${USE_GOLD_LINKER=false}
# Override the default compiler by setting a path to the new compiler. The default
-# compiler depends on USE_SYSTEM_GCC, DISABLE_IMPALA_TOOLCHAIN, and IMPALA_GCC_VERSION.
-# The intended use case is to set the compiler to distcc, in that case the user would
-# also set IMPALA_BUILD_THREADS to increase parallelism.
+# compiler depends on USE_SYSTEM_GCC and IMPALA_GCC_VERSION. The intended use case
+# is to set the compiler to distcc, in that case the user would also set
+# IMPALA_BUILD_THREADS to increase parallelism.
: ${IMPALA_CXX_COMPILER=default}
# If enabled, debug symbols are added to cross-compiled IR.
: ${ENABLE_IMPALA_IR_DEBUG_INFO=false}
-export DISABLE_IMPALA_TOOLCHAIN
export IMPALA_TOOLCHAIN
export USE_SYSTEM_GCC
export USE_GOLD_LINKER
@@ -95,9 +101,7 @@ export USE_KUDU_DEBUG_BUILD
if [[ -z "${KUDU_IS_SUPPORTED-}" ]]; then
KUDU_IS_SUPPORTED=true
if [[ -z $KUDU_BUILD_DIR ]]; then
- if [[ $DISABLE_IMPALA_TOOLCHAIN -eq 1 ]]; then
- KUDU_IS_SUPPORTED=false
- elif ! $IS_OSX; then
+ if ! $IS_OSX; then
if ! which lsb_release &>/dev/null; then
echo Unable to find the 'lsb_release' command. \
Please ensure it is available in your PATH. 1>&2
@@ -226,47 +230,35 @@ export NUM_CONCURRENT_TESTS=${NUM_CONCURRENT_TESTS-${CORES}}
# Versions of toolchain dependencies (or if toolchain is not used of dependencies in
# thirdparty)
-export IMPALA_AVRO_VERSION=1.7.4
+export IMPALA_AVRO_VERSION=1.7.4-p4
export IMPALA_BINUTILS_VERSION=2.26
export IMPALA_BOOST_VERSION=1.57.0
export IMPALA_BREAKPAD_VERSION=20150612-p1
-export IMPALA_BZIP2_VERSION=1.0.6
+export IMPALA_BZIP2_VERSION=1.0.6-p1
export IMPALA_CYRUS_SASL_VERSION=2.1.23
export IMPALA_GCC_VERSION=4.9.2
export IMPALA_GFLAGS_VERSION=2.0
-export IMPALA_GLOG_VERSION=0.3.2
+export IMPALA_GLOG_VERSION=0.3.2-p1
export IMPALA_GPERFTOOLS_VERSION=2.5
export IMPALA_GTEST_VERSION=1.6.0
export IMPALA_KUDU_VERSION=0.8.0-RC1
-export IMPALA_LLVM_VERSION=3.8.0
-export IMPALA_LLVM_DEBUG_VERSION=3.8.0
-export IMPALA_LLVM_ASAN_VERSION=3.8.0
+export IMPALA_LLVM_VERSION=3.8.0-p1
+export IMPALA_LLVM_ASAN_VERSION=3.8.0-p1
+# Debug builds should use the release+asserts build to get additional coverage.
+# Don't use the LLVM debug build because the binaries are too large to distribute.
+export IMPALA_LLVM_DEBUG_VERSION=3.8.0-asserts-p1
export IMPALA_LZ4_VERSION=svn
export IMPALA_OPENLDAP_VERSION=2.4.25
export IMPALA_OPENSSL_VERSION=0.9.8zf
export IMPALA_POSTGRES_JDBC_DRIVER_VERSION=9.0-801
export IMPALA_RAPIDJSON_VERSION=0.11
-export IMPALA_RE2_VERSION=20130115
+export IMPALA_RE2_VERSION=20130115-p1
export IMPALA_SNAPPY_VERSION=1.1.3
export IMPALA_SQUEASEL_VERSION=3.3
-export IMPALA_THRIFT_VERSION=0.9.0
+export IMPALA_THRIFT_VERSION=0.9.0-p8
export IMPALA_THRIFT_JAVA_VERSION=0.9.0
export IMPALA_ZLIB_VERSION=1.2.8
-# Some of the variables need to be overwritten to explicitly mark the patch level
-if [[ -n "$IMPALA_TOOLCHAIN" ]]; then
- IMPALA_AVRO_VERSION+=-p4
- IMPALA_BZIP2_VERSION+=-p1
- IMPALA_GLOG_VERSION+=-p1
- IMPALA_THRIFT_VERSION+=-p8
- IMPALA_RE2_VERSION+=-p1
- IMPALA_LLVM_VERSION+=-p1
- IMPALA_LLVM_ASAN_VERSION+=-p1
- # Debug builds should use the release+asserts build to get additional coverage.
- # Don't use the LLVM debug build because the binaries are too large to distribute.
- IMPALA_LLVM_DEBUG_VERSION+=-asserts-p1
-fi
-
export KUDU_MASTER=${KUDU_MASTER:-"127.0.0.1"}
export KUDU_MASTER_PORT=${KUDU_MASTER_PORT:-"7051"}
# TODO: Figure out a way to use a snapshot version without causing a lot of breakage due
@@ -282,20 +274,6 @@ if [[ $OSTYPE == "darwin"* ]]; then
IMPALA_THRIFT_JAVA_VERSION=0.9.2
fi
-
-if [[ ! -z "${IMPALA_CYRUS_SASL_INSTALL_DIR:-}" ]]
-then
- export IMPALA_CYRUS_SASL_INSTALL_DIR # Ensure it's exported
-elif [[ "${IMPALA_HOME}" =~ "~" ]]
-then
- # Sasl has problems with 'make install' if the path contains a ~, e.g.
- # /some~directory/impala. In our packaging jobs, the path contains ~ so we'll
- # just install somewhere else as a workaround.
- export IMPALA_CYRUS_SASL_INSTALL_DIR=/tmp/impala-build/cyrus-sasl-${IMPALA_CYRUS_SASL_VERSION}/build
-else
- export IMPALA_CYRUS_SASL_INSTALL_DIR=${IMPALA_HOME}/thirdparty/cyrus-sasl-${IMPALA_CYRUS_SASL_VERSION}/build
-fi
-
export IMPALA_HADOOP_VERSION=2.6.0-cdh5.8.0-SNAPSHOT
export IMPALA_HBASE_VERSION=1.2.0-cdh5.8.0-SNAPSHOT
export IMPALA_HIVE_VERSION=1.1.0-cdh5.8.0-SNAPSHOT
@@ -313,8 +291,11 @@ export IMPALA_AUX_DATASET_DIR=$IMPALA_AUX_TEST_HOME/testdata/datasets
export IMPALA_COMMON_DIR=$IMPALA_HOME/common
export PATH=$IMPALA_HOME/bin:$PATH
+# The directory in which all the thirdparty CDH components live.
+CDH_COMPONENTS_HOME=$IMPALA_HOME/thirdparty
+
# Hadoop dependencies are snapshots in the Impala tree
-export HADOOP_HOME=$IMPALA_HOME/thirdparty/hadoop-${IMPALA_HADOOP_VERSION}/
+export HADOOP_HOME=$CDH_COMPONENTS_HOME/hadoop-${IMPALA_HADOOP_VERSION}/
export HADOOP_CONF_DIR=$IMPALA_FE_DIR/src/test/resources
: ${HADOOP_CLASSPATH=}
@@ -326,12 +307,12 @@ HADOOP_CLASSPATH+=":$LZO_JAR_PATH"
export MINI_DFS_BASE_DATA_DIR=$IMPALA_HOME/cdh-${CDH_MAJOR_VERSION}-hdfs-data
export PATH=$HADOOP_HOME/bin:$PATH
-export LLAMA_HOME=$IMPALA_HOME/thirdparty/llama-${IMPALA_LLAMA_VERSION}/
-export MINIKDC_HOME=$IMPALA_HOME/thirdparty/llama-minikdc-${IMPALA_MINIKDC_VERSION}
-export SENTRY_HOME=$IMPALA_HOME/thirdparty/sentry-${IMPALA_SENTRY_VERSION}
+export LLAMA_HOME=$CDH_COMPONENTS_HOME/llama-${IMPALA_LLAMA_VERSION}/
+export MINIKDC_HOME=$CDH_COMPONENTS_HOME/llama-minikdc-${IMPALA_MINIKDC_VERSION}
+export SENTRY_HOME=$CDH_COMPONENTS_HOME/sentry-${IMPALA_SENTRY_VERSION}
export SENTRY_CONF_DIR=$IMPALA_HOME/fe/src/test/resources
-export HIVE_HOME=$IMPALA_HOME/thirdparty/hive-${IMPALA_HIVE_VERSION}/
+export HIVE_HOME=$CDH_COMPONENTS_HOME/hive-${IMPALA_HIVE_VERSION}/
export PATH=$HIVE_HOME/bin:$PATH
export HIVE_CONF_DIR=$IMPALA_FE_DIR/src/test/resources
@@ -346,7 +327,7 @@ export AUX_CLASSPATH="${LZO_JAR_PATH}"
### Tell hive not to use jline
export HADOOP_USER_CLASSPATH_FIRST=true
-export HBASE_HOME=$IMPALA_HOME/thirdparty/hbase-${IMPALA_HBASE_VERSION}/
+export HBASE_HOME=$CDH_COMPONENTS_HOME/hbase-${IMPALA_HBASE_VERSION}/
export PATH=$HBASE_HOME/bin:$PATH
# Add the jars so hive can create hbase tables.
@@ -358,18 +339,11 @@ export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-hadoop-compat-${IMPALA
export HBASE_CONF_DIR=$HIVE_CONF_DIR
-# Optionally set the Thrift home to the toolchain
-if [[ -z $IMPALA_TOOLCHAIN ]]; then
- THRIFT_SRC_DIR=${IMPALA_HOME}/thirdparty/thrift-${IMPALA_THRIFT_VERSION}
- export THRIFT_HOME=${THRIFT_SRC_DIR}/build
-else
- export THRIFT_HOME=${IMPALA_TOOLCHAIN}/thrift-${IMPALA_THRIFT_VERSION}
-fi
+# Set $THRIFT_HOME to the Thrift directory in toolchain.
+export THRIFT_HOME=${IMPALA_TOOLCHAIN}/thrift-${IMPALA_THRIFT_VERSION}
# ASAN needs a matching version of llvm-symbolizer to symbolize stack traces.
-if [[ -n $IMPALA_TOOLCHAIN ]]; then
- export ASAN_SYMBOLIZER_PATH=${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_ASAN_VERSION}/bin/llvm-symbolizer
-fi
+export ASAN_SYMBOLIZER_PATH=${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_ASAN_VERSION}/bin/llvm-symbolizer
export CLUSTER_DIR=${IMPALA_HOME}/testdata/cluster
@@ -399,11 +373,7 @@ export LIBHDFS_OPTS="${LIBHDFS_OPTS}:${IMPALA_HOME}/be/build/debug/service"
export ARTISTIC_STYLE_OPTIONS=$IMPALA_BE_DIR/.astylerc
-if [[ -z "${IMPALA_TOOLCHAIN}" ]]; then
- IMPALA_SNAPPY_PATH=${IMPALA_HOME}/thirdparty/snappy-${IMPALA_SNAPPY_VERSION}/build/lib
-else
- IMPALA_SNAPPY_PATH=${IMPALA_TOOLCHAIN}/snappy-${IMPALA_SNAPPY_VERSION}/lib
-fi
+export IMPALA_SNAPPY_PATH=${IMPALA_TOOLCHAIN}/snappy-${IMPALA_SNAPPY_VERSION}/lib
export JAVA_LIBRARY_PATH=${IMPALA_SNAPPY_PATH}
@@ -418,10 +388,11 @@ LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JAVA}`:`dirname ${LIB_JSIG}`"
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JVM}`:`dirname ${LIB_HDFS}`"
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_HOME}/be/build/debug/service"
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_SNAPPY_PATH}"
-LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$IMPALA_LZO/build"
+LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_LZO}/build"
-if [[ -n "$IMPALA_TOOLCHAIN" ]]; then
- LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_TOOLCHAIN}/gcc-${IMPALA_GCC_VERSION}/lib64"
+if [ $USE_SYSTEM_GCC -eq 0 ]; then
+ IMPALA_TOOLCHAIN_GCC_LIB=${IMPALA_TOOLCHAIN}/gcc-${IMPALA_GCC_VERSION}/lib64
+ LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_TOOLCHAIN_GCC_LIB}"
fi
export LD_LIBRARY_PATH
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/buildall.sh
----------------------------------------------------------------------
diff --git a/buildall.sh b/buildall.sh
index 45df6cc..2c49b6f 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -244,16 +244,6 @@ if [ $CLEAN_ACTION -eq 1 ]; then
$IMPALA_HOME/bin/clean.sh
fi
-# Only build thirdparty if no toolchain is set
-if [[ -z $IMPALA_TOOLCHAIN ]]; then
- # Sanity check that thirdparty is built.
- if [ ! -e $IMPALA_HOME/thirdparty/gflags-${IMPALA_GFLAGS_VERSION}/libgflags.la ]
- then
- echo "Couldn't find thirdparty build files. Building thirdparty."
- $IMPALA_HOME/bin/build_thirdparty.sh $([ ${CLEAN_ACTION} -eq 0 ] && echo '-noclean')
- fi
-fi
-
MAKE_IMPALA_ARGS="${MAKE_IMPALA_ARGS} -build_type=${CMAKE_BUILD_TYPE}"
if [ $BUILD_FE_ONLY -eq 1 ]; then
@@ -308,9 +298,7 @@ $IMPALA_HOME/bin/make_impala.sh ${MAKE_IMPALA_ARGS}
if [ -e $IMPALA_LZO ]
then
pushd $IMPALA_LZO
- if [[ ! -z $IMPALA_TOOLCHAIN ]]; then
- LZO_CMAKE_ARGS+=" -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake"
- fi
+ LZO_CMAKE_ARGS+=" -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake"
rm -f CMakeCache.txt
cmake ${LZO_CMAKE_ARGS}
${MAKE_CMD}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/cmake_modules/FindPICLibs.cmake
----------------------------------------------------------------------
diff --git a/cmake_modules/FindPICLibs.cmake b/cmake_modules/FindPICLibs.cmake
deleted file mode 100644
index edbf4fd..0000000
--- a/cmake_modules/FindPICLibs.cmake
+++ /dev/null
@@ -1,19 +0,0 @@
-# In order to statically link in the Boost, bz2, event and z libraries, they
-# needs to be recompiled with -fPIC. Set $PIC_LIB_PATH to the location of
-# these libraries in the environment, or dynamic linking will be used instead.
-
-# If the toolchain is present, we have all the libraries compiled with -fpic, so we don't
-# need this check
-
-if (NOT IMPALA_TOOLCHAIN)
- IF (DEFINED ENV{PIC_LIB_PATH})
- set(CMAKE_SKIP_RPATH TRUE)
- set(Boost_USE_STATIC_LIBS ON)
- set(Boost_USE_STATIC_RUNTIME ON)
- set(LIBBZ2 $ENV{PIC_LIB_PATH}/lib/libbz2.a)
- set(LIBZ $ENV{PIC_LIB_PATH}/lib/libz.a)
- ELSE (DEFINED ENV{PIC_LIB_PATH})
- set(Boost_USE_STATIC_LIBS OFF)
- set(Boost_USE_STATIC_RUNTIME OFF)
- ENDIF (DEFINED ENV{PIC_LIB_PATH})
-endif()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/cmake_modules/asan_toolchain.cmake
----------------------------------------------------------------------
diff --git a/cmake_modules/asan_toolchain.cmake b/cmake_modules/asan_toolchain.cmake
index 05997bf..de6a338 100644
--- a/cmake_modules/asan_toolchain.cmake
+++ b/cmake_modules/asan_toolchain.cmake
@@ -12,8 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set(IMPALA_TOOLCHAIN ON)
-
# Set the root directory for the toolchain
set(TOOLCHAIN_ROOT $ENV{IMPALA_TOOLCHAIN})
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/86ff18ee/cmake_modules/toolchain.cmake
----------------------------------------------------------------------
diff --git a/cmake_modules/toolchain.cmake b/cmake_modules/toolchain.cmake
index 86ebf9d..7cb5119 100644
--- a/cmake_modules/toolchain.cmake
+++ b/cmake_modules/toolchain.cmake
@@ -12,8 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set(IMPALA_TOOLCHAIN ON)
-
# Set the root directory for the toolchain
set(TOOLCHAIN_ROOT $ENV{IMPALA_TOOLCHAIN})