You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jr...@apache.org on 2018/01/19 21:41:33 UTC
[4/8] impala git commit: Revert "IMPALA-5528: Upgrade GPerfTools to
2.6.3 and tune TCMalloc for KRPC"
Revert "IMPALA-5528: Upgrade GPerfTools to 2.6.3 and tune TCMalloc for KRPC"
This reverts commit df3a440fff38225a03879955c99a87d8ced3b13a.
Apparently, linking Impalad against GPerfTools 2.6.3 caused Impalad to fail
on certain platforms (OLE6). The failure's symptom is SIGSEGV when trying to
exec Impalad binary. It's unclear which commit in GPerfTools could have caused
it so backing up this change to allow Impala to unbreak some platforms for now.
Change-Id: I97cccca74fb199d6ff0a42fe818f8789a0d66e83
Reviewed-on: http://gerrit.cloudera.org:8080/9057
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/b3d38b5c
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/b3d38b5c
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/b3d38b5c
Branch: refs/heads/2.x
Commit: b3d38b5c8650fb455ca556f321fb7aea35dbd5ee
Parents: 028a83e
Author: Michael Ho <kw...@cloudera.com>
Authored: Wed Jan 17 19:27:17 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Thu Jan 18 23:25:09 2018 +0000
----------------------------------------------------------------------
.../runtime/bufferpool/buffer-allocator-test.cc | 7 -------
be/src/runtime/bufferpool/free-list-test.cc | 13 +-----------
be/src/runtime/bufferpool/suballocator-test.cc | 13 +-----------
be/src/runtime/exec-env.cc | 21 ++++++++++----------
bin/impala-config.sh | 4 ++--
5 files changed, 14 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/b3d38b5c/be/src/runtime/bufferpool/buffer-allocator-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/bufferpool/buffer-allocator-test.cc b/be/src/runtime/bufferpool/buffer-allocator-test.cc
index 2b87278..21a9c08 100644
--- a/be/src/runtime/bufferpool/buffer-allocator-test.cc
+++ b/be/src/runtime/bufferpool/buffer-allocator-test.cc
@@ -22,8 +22,6 @@
#include "runtime/bufferpool/buffer-pool-internal.h"
#include "runtime/bufferpool/buffer-pool.h"
#include "runtime/bufferpool/system-allocator.h"
-#include "runtime/test-env.h"
-#include "service/fe-support.h"
#include "testutil/cpu-util.h"
#include "testutil/gtest-util.h"
#include "util/cpu-info.h"
@@ -41,8 +39,6 @@ using BufferHandle = BufferPool::BufferHandle;
class BufferAllocatorTest : public ::testing::Test {
public:
virtual void SetUp() {
- test_env_.reset(new TestEnv);
- ASSERT_OK(test_env_->Init());
dummy_pool_ = obj_pool_.Add(new BufferPool(1, 0, 0));
dummy_reservation_.InitRootTracker(nullptr, 0);
ASSERT_OK(dummy_pool_->RegisterClient("", nullptr, &dummy_reservation_, nullptr, 0,
@@ -63,8 +59,6 @@ class BufferAllocatorTest : public ::testing::Test {
/// The minimum buffer size used in most tests.
const static int64_t TEST_BUFFER_LEN = 1024;
- boost::scoped_ptr<TestEnv> test_env_;
-
ObjectPool obj_pool_;
/// Need a dummy pool and client to pass around. We bypass the reservation mechanisms
@@ -206,7 +200,6 @@ TEST_F(SystemAllocatorTest, LargeAllocFailure) {
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
impala::InitCommonRuntime(argc, argv, true, impala::TestInfo::BE_TEST);
- impala::InitFeSupport();
int result = 0;
for (bool mmap : {false, true}) {
for (bool madvise : {false, true}) {
http://git-wip-us.apache.org/repos/asf/impala/blob/b3d38b5c/be/src/runtime/bufferpool/free-list-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/bufferpool/free-list-test.cc b/be/src/runtime/bufferpool/free-list-test.cc
index d3c4c9a..7cb80b4 100644
--- a/be/src/runtime/bufferpool/free-list-test.cc
+++ b/be/src/runtime/bufferpool/free-list-test.cc
@@ -21,8 +21,6 @@
#include "common/object-pool.h"
#include "runtime/bufferpool/free-list.h"
#include "runtime/bufferpool/system-allocator.h"
-#include "runtime/test-env.h"
-#include "service/fe-support.h"
#include "testutil/gtest-util.h"
#include "testutil/rand-util.h"
@@ -33,8 +31,6 @@ namespace impala {
class FreeListTest : public ::testing::Test {
protected:
virtual void SetUp() override {
- test_env_.reset(new TestEnv);
- ASSERT_OK(test_env_->Init());
allocator_ = obj_pool_.Add(new SystemAllocator(MIN_BUFFER_LEN));
RandTestUtil::SeedRng("FREE_LIST_TEST_SEED", &rng_);
}
@@ -75,8 +71,6 @@ class FreeListTest : public ::testing::Test {
const static int MIN_BUFFER_LEN = 1024;
- boost::scoped_ptr<TestEnv> test_env_;
-
/// Per-test random number generator. Seeded before every test.
std::mt19937 rng_;
@@ -163,9 +157,4 @@ TEST_F(FreeListTest, ReturnOrder) {
}
}
-int main(int argc, char** argv) {
- ::testing::InitGoogleTest(&argc, argv);
- impala::InitCommonRuntime(argc, argv, true, impala::TestInfo::BE_TEST);
- impala::InitFeSupport();
- return RUN_ALL_TESTS();
-}
+IMPALA_TEST_MAIN();
http://git-wip-us.apache.org/repos/asf/impala/blob/b3d38b5c/be/src/runtime/bufferpool/suballocator-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/bufferpool/suballocator-test.cc b/be/src/runtime/bufferpool/suballocator-test.cc
index 470b065..6cd53fb 100644
--- a/be/src/runtime/bufferpool/suballocator-test.cc
+++ b/be/src/runtime/bufferpool/suballocator-test.cc
@@ -27,8 +27,6 @@
#include "common/object-pool.h"
#include "runtime/bufferpool/reservation-tracker.h"
#include "runtime/bufferpool/suballocator.h"
-#include "runtime/test-env.h"
-#include "service/fe-support.h"
#include "testutil/death-test-util.h"
#include "testutil/gtest-util.h"
#include "testutil/rand-util.h"
@@ -46,8 +44,6 @@ namespace impala {
class SuballocatorTest : public ::testing::Test {
public:
virtual void SetUp() override {
- test_env_.reset(new TestEnv);
- ASSERT_OK(test_env_->Init());
RandTestUtil::SeedRng("SUBALLOCATOR_TEST_SEED", &rng_);
profile_ = RuntimeProfile::Create(&obj_pool_, "test profile");
}
@@ -115,8 +111,6 @@ class SuballocatorTest : public ::testing::Test {
/// Clients for the buffer pool. Deregistered and freed after every test.
vector<unique_ptr<BufferPool::ClientHandle>> clients_;
- boost::scoped_ptr<TestEnv> test_env_;
-
/// Global profile - recreated for every test.
RuntimeProfile* profile_;
@@ -368,9 +362,4 @@ void SuballocatorTest::AssertMemoryValid(
}
}
-int main(int argc, char** argv) {
- ::testing::InitGoogleTest(&argc, argv);
- impala::InitCommonRuntime(argc, argv, true, impala::TestInfo::BE_TEST);
- impala::InitFeSupport();
- return RUN_ALL_TESTS();
-}
+IMPALA_TEST_MAIN();
http://git-wip-us.apache.org/repos/asf/impala/blob/b3d38b5c/be/src/runtime/exec-env.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc
index 94ca834..6d9a857 100644
--- a/be/src/runtime/exec-env.cc
+++ b/be/src/runtime/exec-env.cc
@@ -271,6 +271,16 @@ Status ExecEnv::Init() {
"bytes value or percentage: $0", FLAGS_mem_limit));
}
+#if !defined(ADDRESS_SANITIZER) && !defined(THREAD_SANITIZER)
+ // Aggressive decommit is required so that unused pages in the TCMalloc page heap are
+ // not backed by physical pages and do not contribute towards memory consumption.
+ // Enable it in TCMalloc before InitBufferPool().
+ if (!MallocExtension::instance()->SetNumericProperty(
+ "tcmalloc.aggressive_memory_decommit", 1)) {
+ return Status("Failed to enable TCMalloc aggressive decommit.");
+ }
+#endif
+
if (!BitUtil::IsPowerOf2(FLAGS_min_buffer_size)) {
return Status(Substitute(
"--min_buffer_size must be a power-of-two: $0", FLAGS_min_buffer_size));
@@ -310,11 +320,6 @@ Status ExecEnv::Init() {
FLAGS_datastream_service_num_svc_threads : CpuInfo::num_cores();
RETURN_IF_ERROR(rpc_mgr_->RegisterService(num_svc_threads,
FLAGS_datastream_service_queue_depth, move(data_svc)));
- // Bump thread cache to 1GB to reduce contention for TCMalloc central
- // list's spinlock.
- if (FLAGS_tcmalloc_max_total_thread_cache_bytes == 0) {
- FLAGS_tcmalloc_max_total_thread_cache_bytes = 1 << 30;
- }
}
mem_tracker_.reset(
@@ -427,12 +432,6 @@ Status ExecEnv::StartKrpcService() {
void ExecEnv::InitBufferPool(int64_t min_buffer_size, int64_t capacity,
int64_t clean_pages_limit) {
-#if !defined(ADDRESS_SANITIZER) && !defined(THREAD_SANITIZER)
- // Aggressive decommit is required so that unused pages in the TCMalloc page heap are
- // not backed by physical pages and do not contribute towards memory consumption.
- MallocExtension::instance()->SetNumericProperty(
- "tcmalloc.aggressive_memory_decommit", 1);
-#endif
buffer_pool_.reset(new BufferPool(min_buffer_size, capacity, clean_pages_limit));
buffer_reservation_.reset(new ReservationTracker());
buffer_reservation_->InitRootTracker(nullptr, capacity);
http://git-wip-us.apache.org/repos/asf/impala/blob/b3d38b5c/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index f5b67a9..058dc01 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -72,7 +72,7 @@ fi
# moving to a different build of the toolchain, e.g. when a version is bumped or a
# compile option is changed. The build id can be found in the output of the toolchain
# build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=4-7847dc86c4
+export IMPALA_TOOLCHAIN_BUILD_ID=482-c2361403fc
# Versions of toolchain dependencies.
# -----------------------------------
export IMPALA_AVRO_VERSION=1.7.4-p4
@@ -99,7 +99,7 @@ export IMPALA_GFLAGS_VERSION=2.2.0-p1
unset IMPALA_GFLAGS_URL
export IMPALA_GLOG_VERSION=0.3.4-p2
unset IMPALA_GLOG_URL
-export IMPALA_GPERFTOOLS_VERSION=2.6.3
+export IMPALA_GPERFTOOLS_VERSION=2.5
unset IMPALA_GPERFTOOLS_URL
export IMPALA_GTEST_VERSION=1.6.0
unset IMPALA_GTEST_URL