Repository: incubator-impala
Updated Branches:
refs/heads/master b3cbc960a -> 0a654b318
Increase wait times for startup of Hive and its Metastore
On Ubuntu 14.04 on AWS EC2 m4.4x, instances, these components
frequently take more than 30 seconds to start. I have seen the HMS
take more than 90 seconds; this patch sets a more conservative timeout
default.
Change-Id: I43eb8646cca495578c8f9730faa04812957d2917
Reviewed-on: http://gerrit.cloudera.org:8080/5068
Reviewed-by: Jim Apple <jb...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/4b774880
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/4b774880
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/4b774880
Branch: refs/heads/master
Commit: 4b774880c9bd46fccf8855f383ce579e585c7115
Parents: b3cbc96
Author: Jim Apple <jb...@cloudera.com>
Authored: Mon Nov 14 09:38:32 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 15 20:35:01 2016 +0000
----------------------------------------------------------------------
testdata/bin/wait-for-hiveserver2.py | 2 +-
testdata/bin/wait-for-metastore.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b774880/testdata/bin/wait-for-hiveserver2.py
----------------------------------------------------------------------
diff --git a/testdata/bin/wait-for-hiveserver2.py b/testdata/bin/wait-for-hiveserver2.py
index 233b541..bf57ea8 100755
--- a/testdata/bin/wait-for-hiveserver2.py
+++ b/testdata/bin/wait-for-hiveserver2.py
@@ -60,7 +60,7 @@ hs2_client = TCLIService.Client(protocol)
# Try to connect to the HiveServer2 service and create a session
now = time.time()
-TIMEOUT_SECONDS = 30.0
+TIMEOUT_SECONDS = 300.0
while time.time() - now < TIMEOUT_SECONDS:
try:
hs2_transport.open()
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b774880/testdata/bin/wait-for-metastore.py
----------------------------------------------------------------------
diff --git a/testdata/bin/wait-for-metastore.py b/testdata/bin/wait-for-metastore.py
index a528142..2dc6550 100755
--- a/testdata/bin/wait-for-metastore.py
+++ b/testdata/bin/wait-for-metastore.py
@@ -50,7 +50,7 @@ hive_client = ThriftHiveMetastore.Client(protocol)
# Try to connect to the Hive metastore
now = time.time()
-TIMEOUT_SECONDS = 30.0
+TIMEOUT_SECONDS = 300.0
while time.time() - now < TIMEOUT_SECONDS:
try:
hive_transport.open()
Run MT_DOP tests on all file formats.
Change-Id: I28d5bcc48bbe32fb970b41daa919096061a05beb
Reviewed-on: http://gerrit.cloudera.org:8080/5025
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/0a654b31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/0a654b31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/0a654b31
Branch: refs/heads/master
Commit: 0a654b31869c77d667cd692a5f36cf0748b31c64
Parents: 9434e38
Author: Alex Behm <al...@cloudera.com>
Authored: Wed Nov 9 12:06:26 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 15 23:47:40 2016 +0000
----------------------------------------------------------------------
tests/query_test/test_mt_dop.py | 44 ++++++++++++++++++++++++------------
1 file changed, 30 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0a654b31/tests/query_test/test_mt_dop.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_mt_dop.py b/tests/query_test/test_mt_dop.py
index 4bc5179..d05d6b4 100644
--- a/tests/query_test/test_mt_dop.py
+++ b/tests/query_test/test_mt_dop.py
@@ -31,11 +31,6 @@ class TestMtDop(ImpalaTestSuite):
def add_test_dimensions(cls):
super(TestMtDop, cls).add_test_dimensions()
cls.TestMatrix.add_dimension(TestDimension('mt_dop', *MT_DOP_VALUES))
- # IMPALA-4332: The MT scheduler does not work for Kudu or HBase tables.
- cls.TestMatrix.add_constraint(\
- lambda v: v.get_value('table_format').file_format != 'hbase')
- cls.TestMatrix.add_constraint(\
- lambda v: v.get_value('table_format').file_format != 'kudu')
@classmethod
def get_workload(cls):
@@ -46,16 +41,37 @@ class TestMtDop(ImpalaTestSuite):
self.run_test_case('QueryTest/mt-dop', vector)
def test_compute_stats(self, unique_database, vector):
- table_loc = self._get_table_location("alltypes", vector)
- # Create a second table in the same format pointing to the same data files.
- # This function switches to the format-specific DB in vector.
- self.execute_query_using_client(self.client,
- "create external table %s.mt_dop like alltypes location '%s'"
- % (unique_database, table_loc), vector)
- self.execute_query_using_client(self.client,
- "alter table %s.mt_dop recover partitions" % unique_database, vector)
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
- self.run_test_case('QueryTest/mt-dop-compute-stats', vector, unique_database)
+ file_format = vector.get_value('table_format').file_format
+ fq_table_name = "%s.mt_dop" % unique_database
+
+ # Different formats need different DDL for creating a test table, and also
+ # have different expected results for compute stats.
+ expected_results = None
+ if file_format == 'kudu':
+ # CREATE TABLE LIKE is currently not supported for Kudu tables.
+ self.execute_query("create external table %s stored as kudu "
+ "tblproperties('kudu.table_name'='impala::functional_kudu.alltypes')"
+ % fq_table_name)
+ expected_results = "Updated 1 partition(s) and 13 column(s)."
+ elif file_format == 'hbase':
+ self.execute_query(
+ "create external table %s like functional_hbase.alltypes" % fq_table_name)
+ expected_results = "Updated 1 partition(s) and 13 column(s)."
+ else:
+ # Create a second table in the same format pointing to the same data files.
+ # This function switches to the format-specific DB in 'vector'.
+ table_loc = self._get_table_location("alltypes", vector)
+ self.execute_query_using_client(self.client,
+ "create external table %s like alltypes location '%s'"
+ % (fq_table_name, table_loc), vector)
+ # Recover partitions for HDFS tables.
+ self.execute_query("alter table %s recover partitions" % fq_table_name)
+ expected_results = "Updated 24 partition(s) and 11 column(s)."
+
+ results = self.execute_query("compute stats %s" % fq_table_name,
+ vector.get_value('exec_option'))
+ assert expected_results in results.data
class TestMtDopParquet(ImpalaTestSuite):
@classmethod
clang-tidy should tidy tests; fix alignas error in clang builds. run_clang_tidy.sh was mistakenly using -notests, which doesn't even compile the tests, rather than -skiptests, which compiles (but does not run) the backend tests. When I discovered this, I also found that all clang builds (including tidy and asan) had been broken by my previous alignas commit (10a4c5a2e47ec7bf1e563b6a75d1bf2afb47ebd8). This patch fixes that as well. Change-Id: Ib7066039f78d7ee039db619b96e25665b4d63503 Reviewed-on: http://gerrit.cloudera.org:8080/5094 Reviewed-by: Sailesh Mukil <sa...@cloudera.com> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/9434e38a Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/9434e38a Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/9434e38a Branch: refs/heads/master Commit: 9434e38abb515da25a3adbe43a217d4361ee531c Parents: 38ee3b6 Author: Jim Apple <jb...@cloudera.com> Authored: Tue Nov 15 06:22:13 2016 -0800 Committer: Internal Jenkins <cl...@gerrit.cloudera.org> Committed: Tue Nov 15 23:33:45 2016 +0000 ---------------------------------------------------------------------- be/src/runtime/mem-pool.cc | 2 +- bin/run_clang_tidy.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9434e38a/be/src/runtime/mem-pool.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/mem-pool.cc b/be/src/runtime/mem-pool.cc index 5f3feca..0ca3822 100644 --- a/be/src/runtime/mem-pool.cc +++ b/be/src/runtime/mem-pool.cc @@ -37,7 +37,7 @@ const int MemPool::MAX_CHUNK_SIZE; const char* MemPool::LLVM_CLASS_NAME = "class.impala::MemPool"; const int MemPool::DEFAULT_ALIGNMENT; -uint32_t MemPool::zero_length_region_ = MEM_POOL_POISON; +uint32_t MemPool::zero_length_region_ alignas(std::max_align_t) = MEM_POOL_POISON; MemPool::MemPool(MemTracker* mem_tracker) : current_chunk_idx_(-1), http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9434e38a/bin/run_clang_tidy.sh ---------------------------------------------------------------------- diff --git a/bin/run_clang_tidy.sh b/bin/run_clang_tidy.sh index aa95845..7d42597 100755 --- a/bin/run_clang_tidy.sh +++ b/bin/run_clang_tidy.sh @@ -30,7 +30,7 @@ set -euo pipefail echo "Compiling" -if ! ./buildall.sh -notests -tidy -so -noclean 1>/dev/null 2>/dev/null +if ! ./buildall.sh -skiptests -tidy -so -noclean 1>/dev/null 2>/dev/null then echo "WARNING: compile failed" >&2 fi
IMPALA-4436: StringValue::StringCompare() should match strncmp()
According to the C standard, strncmp() interprets characters as
unsigned, whereas StringCompare() uses char (which happens to be
signed). This means that for values greater than 127, they don't give
the same result (which is especially bad considering StringCompare()
falls back to strncmp(), and so the answer depends on the mismatched
position).
Fix StringCompare() to interpret as unsigned char.
Change-Id: Ic0750f98d8c5ef7d0c0ea279cd1f80b4acbad1be
Reviewed-on: http://gerrit.cloudera.org:8080/5083
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/6937fa9a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/6937fa9a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/6937fa9a
Branch: refs/heads/master
Commit: 6937fa9a4caa9039a95d661ddf209777caa805f5
Parents: 4b77488
Author: Dan Hecht <dh...@cloudera.com>
Authored: Mon Nov 14 16:47:39 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 15 20:47:18 2016 +0000
----------------------------------------------------------------------
be/src/runtime/CMakeLists.txt | 1 +
be/src/runtime/string-compare-test.cc | 67 ++++++++++++++++++++++++++++++
be/src/runtime/string-value.inline.h | 4 +-
3 files changed, 71 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6937fa9a/be/src/runtime/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
index a0b0a94..19db090 100644
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -78,6 +78,7 @@ ADD_BE_TEST(disk-io-mgr-test)
ADD_BE_TEST(buffered-block-mgr-test)
ADD_BE_TEST(parallel-executor-test)
ADD_BE_TEST(raw-value-test)
+ADD_BE_TEST(string-compare-test)
ADD_BE_TEST(string-search-test)
ADD_BE_TEST(string-value-test)
ADD_BE_TEST(thread-resource-mgr-test)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6937fa9a/be/src/runtime/string-compare-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/string-compare-test.cc b/be/src/runtime/string-compare-test.cc
new file mode 100644
index 0000000..ef81ac6
--- /dev/null
+++ b/be/src/runtime/string-compare-test.cc
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstring>
+
+#include "runtime/string-value.inline.h"
+#include "testutil/gtest-util.h"
+
+#include "common/names.h"
+
+// Test to verify that impala::StringCompare() and clib strncmp() give equivalent results.
+namespace impala {
+
+// Expect that strncmp() and StringCompare() return the same result.
+static void RunTestCase(const string& l, const string& r) {
+ int cmp_len = ::min(l.size(), r.size());
+ EXPECT_EQ(strncmp(l.c_str(), r.c_str(), cmp_len),
+ StringCompare(l.c_str(), l.size(), r.c_str(), r.size(), cmp_len));
+}
+
+TEST(StringCompareTest, Basic) {
+ const struct Cases {
+ string lhs, rhs;
+ } cases[] = {{"abc", "abc"},
+ {"abc", "abd"},
+ {"", ""},
+ {"\x01", "\x01"},
+ {"\x01", "\x02"},
+ {"\x7f", "\x01"},
+ {"\x80", "\x80"},
+ {"\x80", "\x01"},
+ {"\x7F", "\x80"},
+ {"\xFF", "\x01"},
+ {"\xFF", "\x7F"},
+ {"\xE9", "\x32"},
+ {"\x32", "\xE5"},
+ {"\xE5", "\xE9"}};
+
+ for (const Cases& c : cases) {
+ RunTestCase(c.lhs, c.rhs);
+ RunTestCase(c.rhs, c.lhs);
+ }
+
+ // Induce the SSE 4.2 code path by using longer strings.
+ string suffix = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+ for (const Cases& c : cases) {
+ RunTestCase(c.lhs + suffix, c.rhs + suffix);
+ RunTestCase(c.rhs + suffix, c.lhs + suffix);
+ }
+}
+}
+
+IMPALA_TEST_MAIN();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6937fa9a/be/src/runtime/string-value.inline.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/string-value.inline.h b/be/src/runtime/string-value.inline.h
index f52005f..073d01f 100644
--- a/be/src/runtime/string-value.inline.h
+++ b/be/src/runtime/string-value.inline.h
@@ -47,7 +47,9 @@ static inline int StringCompare(const char* s1, int n1, const char* s2, int n2,
SSEUtil::CHARS_PER_128_BIT_REGISTER, xmm1,
SSEUtil::CHARS_PER_128_BIT_REGISTER);
if (chars_match != SSEUtil::CHARS_PER_128_BIT_REGISTER) {
- return s1[chars_match] - s2[chars_match];
+ // Match strncmp() behavior, which interprets characters as unsigned char.
+ return static_cast<unsigned char>(s1[chars_match]) -
+ static_cast<unsigned char>(s2[chars_match]);
}
len -= SSEUtil::CHARS_PER_128_BIT_REGISTER;
s1 += SSEUtil::CHARS_PER_128_BIT_REGISTER;
IMPALA-4444: Transfer row group resources to row batch on scan failure
Previously, if any column reader fails in HdfsParqetScanner::AssembleRows(),
the memory pools associated with the ScratchTupleBatch will be freed. This
is problematic as ScratchTupleBatch may contain memory pools which are still
referenced by row batches shipped upstream. This is possible because memory
pools used by parquet column readers (e.g. decompressor_pool_) won't be
transferred to a ScratchTupleBatch until the data page is exhausted. So,
the memory pools of the previous data page is always attached to the
ScratchTupleBatch of the current data page. On a scan failure, it's not
necessarily safe to free the memory pool attached to the current ScratchTupleBatch.
This patch fixes the problem above by transferring the memory pool and other
resources associated with a row group to the current row batch in the parquet
scanner on scan failure so it can eventually be freed by upstream operators as
the row batch is consumed.
Change-Id: Id70df470e98dd96284fd176bfbb946e9637ad126
Reviewed-on: http://gerrit.cloudera.org:8080/5052
Reviewed-by: Michael Ho <kw...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/38ee3b69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/38ee3b69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/38ee3b69
Branch: refs/heads/master
Commit: 38ee3b6942f3e044555ee399c51cba65f0b35267
Parents: 6937fa9
Author: Michael Ho <kw...@cloudera.com>
Authored: Thu Nov 10 13:33:46 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 15 23:02:50 2016 +0000
----------------------------------------------------------------------
be/src/exec/exec-node.cc | 2 +-
be/src/exec/hdfs-parquet-scanner.cc | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/38ee3b69/be/src/exec/exec-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/exec-node.cc b/be/src/exec/exec-node.cc
index 255217f..1d93626 100644
--- a/be/src/exec/exec-node.cc
+++ b/be/src/exec/exec-node.cc
@@ -429,7 +429,7 @@ Status ExecNode::ExecDebugAction(TExecNodePhase::type phase, RuntimeState* state
return Status::OK();
}
if (debug_action_ == TDebugAction::MEM_LIMIT_EXCEEDED) {
- mem_tracker()->MemLimitExceeded(state, "Debug Action: MEM_LIMIT_EXCEEDED");
+ return mem_tracker()->MemLimitExceeded(state, "Debug Action: MEM_LIMIT_EXCEEDED");
}
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/38ee3b69/be/src/exec/hdfs-parquet-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner.cc b/be/src/exec/hdfs-parquet-scanner.cc
index bd5d65c..6b157aa 100644
--- a/be/src/exec/hdfs-parquet-scanner.cc
+++ b/be/src/exec/hdfs-parquet-scanner.cc
@@ -537,8 +537,9 @@ Status HdfsParquetScanner::AssembleRows(
bool num_tuples_mismatch = c != 0 && last_num_tuples != scratch_batch_->num_tuples;
if (UNLIKELY(!continue_execution || num_tuples_mismatch)) {
// Skipping this row group. Free up all the resources with this row group.
- scratch_batch_->mem_pool()->FreeAll();
+ FlushRowGroupResources(row_batch);
scratch_batch_->num_tuples = 0;
+ DCHECK(scratch_batch_->AtEnd());
*skip_row_group = true;
if (num_tuples_mismatch) {
parse_status_.MergeStatus(Substitute("Corrupt Parquet file '$0': column '$1' "