You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/06/13 23:17:28 UTC
[1/7] incubator-impala git commit: IMPALA-5482: fix git checkout when
workloads are modified
Repository: incubator-impala
Updated Branches:
refs/heads/master e0f69ca1d -> 428b5a1bf
IMPALA-5482: fix git checkout when workloads are modified
When git checkout would overwrite changes, it fails and alerts the
user to do something with the changes. This patch removes any changes
to files induced by the workload copy-and-paste.
Testing: using a patch provided by Lars Volker that touched
testdata/workloads/ (https://gerrit.cloudera.org/#/c/7073/), I was
able to reproduce the problem he saw and see that this patch fixed it.
Change-Id: I9a0d004c353eb4b547aeaf3c56289594326653d7
Reviewed-on: http://gerrit.cloudera.org:8080/7145
Reviewed-by: Lars Volker <lv...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/de9f5230
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/de9f5230
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/de9f5230
Branch: refs/heads/master
Commit: de9f5230eb5580230a62b19fbc9d93a180ef8c24
Parents: e0f69ca
Author: Jim Apple <jb...@apache.org>
Authored: Fri Jun 9 20:17:58 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Sun Jun 11 18:20:22 2017 +0000
----------------------------------------------------------------------
bin/single_node_perf_run.py | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/de9f5230/bin/single_node_perf_run.py
----------------------------------------------------------------------
diff --git a/bin/single_node_perf_run.py b/bin/single_node_perf_run.py
index 74ab944..b9fe095 100755
--- a/bin/single_node_perf_run.py
+++ b/bin/single_node_perf_run.py
@@ -228,6 +228,8 @@ def perf_ab_test(options, args):
if len(args) > 1 and args[1]:
hash_b = get_git_hash_for_name(args[1])
+ # discard any changes created by the previous restore_workloads()
+ sh.git.checkout("--", "testdata/workloads")
build(hash_b, options)
restore_workloads(workload_dir)
start_impala(options.num_impalads)
@@ -298,6 +300,8 @@ def main():
try:
perf_ab_test(options, args)
finally:
+ # discard any changes created by the previous restore_workloads()
+ sh.git.checkout("--", "testdata/workloads")
sh.git.checkout(current_hash)
[3/7] incubator-impala git commit: IMPALA-5031: Don't dereference
uninitialized memory as a bool.
Posted by ta...@apache.org.
IMPALA-5031: Don't dereference uninitialized memory as a bool.
Every bool takes up a byte of memory, but not every byte of memory is
a valid bool. Dereferencing that memory, even if only to write to, it
is undefined behavior.
I tested exhaustive UBSAN and DEBUG. In UBSAN, the error is no longer
present; in both UBSAN and DEBUG, all tests pass
Change-Id: Ibc03401a9aeb966f33298268e53ab1040d007224
Reviewed-on: http://gerrit.cloudera.org:8080/7148
Reviewed-by: Jim Apple <jb...@apache.org>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/9fd46f7d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/9fd46f7d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/9fd46f7d
Branch: refs/heads/master
Commit: 9fd46f7df1f6f02d0db08155d4f49bcadec0ce5f
Parents: 61696f3
Author: Jim Apple <jb...@apache.org>
Authored: Sat Jun 10 10:32:26 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Mon Jun 12 23:54:40 2017 +0000
----------------------------------------------------------------------
be/src/runtime/raw-value.cc | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9fd46f7d/be/src/runtime/raw-value.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/raw-value.cc b/be/src/runtime/raw-value.cc
index 872b356..b7b499d 100644
--- a/be/src/runtime/raw-value.cc
+++ b/be/src/runtime/raw-value.cc
@@ -117,7 +117,11 @@ void RawValue::Write(const void* value, void* dst, const ColumnType& type,
case TYPE_NULL:
break;
case TYPE_BOOLEAN:
- *reinterpret_cast<bool*>(dst) = *reinterpret_cast<const bool*>(value);
+ // Unlike the other scalar types, bool has a limited set of valid values, so if
+ // 'dst' is uninitialized memory and happens to point to a value that is not a valid
+ // bool, then dereferencing it via *reinterpret_cast<bool*>(dst) is undefined
+ // behavior.
+ memcpy(dst, value, sizeof(bool));
break;
case TYPE_TINYINT:
*reinterpret_cast<int8_t*>(dst) = *reinterpret_cast<const int8_t*>(value);
[5/7] incubator-impala git commit: IMPALA-5446: dropped
Sorter::Reset() status
Posted by ta...@apache.org.
IMPALA-5446: dropped Sorter::Reset() status
This patch aligns the sorter's methods closer with the ExecNode methods
and moves the possibly-failing parts of Reset() into Open().
Testing:
Added WARN_UNUSED_RESULT to all the sorter methods that return Status to
prevent similar issues in future.
Add a test that sometimes goes down this code path. It was able to cause
a crash at least once every 5 executions.
Ran an exhaustive build to make sure there were no other regressions.
Change-Id: I7d4f9e93a44531901e663b3f1e18edc514363f74
Reviewed-on: http://gerrit.cloudera.org:8080/7134
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/7a0ee685
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/7a0ee685
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/7a0ee685
Branch: refs/heads/master
Commit: 7a0ee685b846b4300c00eb2dc99c6f9c462bf66e
Parents: 58baca7
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Thu Jun 8 16:38:22 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Jun 13 05:13:44 2017 +0000
----------------------------------------------------------------------
be/src/exec/sort-node.cc | 3 ++-
be/src/runtime/sorter.cc | 19 ++++++++-------
be/src/runtime/sorter.h | 25 +++++++++++---------
.../queries/QueryTest/nested-types-tpch.test | 17 +++++++++++++
4 files changed, 43 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/7a0ee685/be/src/exec/sort-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/sort-node.cc b/be/src/exec/sort-node.cc
index 1a8de83..9660ed3 100644
--- a/be/src/exec/sort-node.cc
+++ b/be/src/exec/sort-node.cc
@@ -54,7 +54,7 @@ Status SortNode::Prepare(RuntimeState* state) {
sorter_.reset(
new Sorter(*less_than_.get(), sort_exec_exprs_.sort_tuple_slot_expr_ctxs(),
&row_descriptor_, mem_tracker(), runtime_profile(), state));
- RETURN_IF_ERROR(sorter_->Init());
+ RETURN_IF_ERROR(sorter_->Prepare());
AddCodegenDisabledMessage(state);
return Status::OK();
}
@@ -75,6 +75,7 @@ Status SortNode::Open(RuntimeState* state) {
RETURN_IF_CANCELLED(state);
RETURN_IF_ERROR(QueryMaintenance(state));
RETURN_IF_ERROR(child(0)->Open(state));
+ RETURN_IF_ERROR(sorter_->Open());
// The child has been opened and the sorter created. Sort the input.
// The final merge is done on-demand as rows are requested in GetNext().
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/7a0ee685/be/src/runtime/sorter.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/sorter.cc b/be/src/runtime/sorter.cc
index 1acdf15..0d41ec4 100644
--- a/be/src/runtime/sorter.cc
+++ b/be/src/runtime/sorter.cc
@@ -1362,13 +1362,12 @@ Sorter::~Sorter() {
DCHECK(merge_output_run_ == NULL);
}
-Status Sorter::Init() {
- DCHECK(unsorted_run_ == NULL) << "Already initialized";
+Status Sorter::Prepare() {
+ DCHECK(in_mem_tuple_sorter_ == NULL) << "Already prepared";
TupleDescriptor* sort_tuple_desc = output_row_desc_->tuple_descriptors()[0];
has_var_len_slots_ = sort_tuple_desc->HasVarlenSlots();
in_mem_tuple_sorter_.reset(new TupleSorter(compare_less_than_,
block_mgr_->max_block_size(), sort_tuple_desc->byte_size(), state_));
- unsorted_run_ = obj_pool_.Add(new Run(this, sort_tuple_desc, true));
initial_runs_counter_ = ADD_COUNTER(profile_, "InitialRunsCreated", TUnit::UNIT);
spilled_runs_counter_ = ADD_COUNTER(profile_, "SpilledRuns", TUnit::UNIT);
@@ -1384,8 +1383,14 @@ Status Sorter::Init() {
RETURN_IF_ERROR(block_mgr_->RegisterClient(Substitute("Sorter ptr=$0", this),
min_buffers_required, false, mem_tracker_, state_, &block_mgr_client_));
+ return Status::OK();
+}
- DCHECK(unsorted_run_ != NULL);
+Status Sorter::Open() {
+ DCHECK(in_mem_tuple_sorter_ != NULL) << "Not prepared";
+ DCHECK(unsorted_run_ == NULL) << "Already open";
+ TupleDescriptor* sort_tuple_desc = output_row_desc_->tuple_descriptors()[0];
+ unsorted_run_ = obj_pool_.Add(new Run(this, sort_tuple_desc, true));
RETURN_IF_ERROR(unsorted_run_->Init());
return Status::OK();
}
@@ -1446,16 +1451,12 @@ Status Sorter::GetNext(RowBatch* output_batch, bool* eos) {
}
}
-Status Sorter::Reset() {
+void Sorter::Reset() {
DCHECK(unsorted_run_ == NULL) << "Cannot Reset() before calling InputDone()";
merger_.reset();
// Free resources from the current runs.
CleanupAllRuns();
obj_pool_.Clear();
- unsorted_run_ = obj_pool_.Add(
- new Run(this, output_row_desc_->tuple_descriptors()[0], true));
- RETURN_IF_ERROR(unsorted_run_->Init());
- return Status::OK();
}
void Sorter::Close() {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/7a0ee685/be/src/runtime/sorter.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/sorter.h b/be/src/runtime/sorter.h
index 54e2d22..d91afa8 100644
--- a/be/src/runtime/sorter.h
+++ b/be/src/runtime/sorter.h
@@ -100,24 +100,27 @@ class Sorter {
~Sorter();
- /// Initialization code, including registration to the block_mgr and the initialization
- /// of the unsorted_run_, both of these may fail.
- Status Init();
+ /// Initial set-up of the sorter for execution. Registers with the block mgr.
+ Status Prepare() WARN_UNUSED_RESULT;
+
+ /// Open the sorter for adding rows. Must be called after Prepare() or Reset() and
+ /// before calling AddBatch().
+ Status Open() WARN_UNUSED_RESULT;
/// Adds a batch of input rows to the current unsorted run.
- Status AddBatch(RowBatch* batch);
+ Status AddBatch(RowBatch* batch) WARN_UNUSED_RESULT;
/// Called to indicate there is no more input. Triggers the creation of merger(s) if
/// necessary.
- Status InputDone();
+ Status InputDone() WARN_UNUSED_RESULT;
/// Get the next batch of sorted output rows from the sorter.
- Status GetNext(RowBatch* batch, bool* eos);
+ Status GetNext(RowBatch* batch, bool* eos) WARN_UNUSED_RESULT;
/// Resets all internal state like ExecNode::Reset().
/// Init() must have been called, AddBatch()/GetNext()/InputDone()
/// may or may not have been called.
- Status Reset();
+ void Reset();
/// Close the Sorter and free resources.
void Close();
@@ -134,7 +137,7 @@ class Sorter {
/// 'sorted_runs_'. The Sorter sets the 'deep_copy_input' flag to true for the
/// merger, since the blocks containing input run data will be deleted as input
/// runs are read.
- Status CreateMerger(int max_num_runs);
+ Status CreateMerger(int max_num_runs) WARN_UNUSED_RESULT;
/// Repeatedly replaces multiple smaller runs in sorted_runs_ with a single larger
/// merged run until there are few enough runs to be merged with a single merger.
@@ -143,15 +146,15 @@ class Sorter {
/// a merge. If the number of sorted runs is too large, merge sets of smaller runs
/// into large runs until a final merge can be performed. An intermediate row batch
/// containing deep copied rows is used for the output of each intermediate merge.
- Status MergeIntermediateRuns();
+ Status MergeIntermediateRuns() WARN_UNUSED_RESULT;
/// Execute a single step of the intermediate merge, pulling rows from 'merger_'
/// and adding them to 'merged_run'.
- Status ExecuteIntermediateMerge(Sorter::Run* merged_run);
+ Status ExecuteIntermediateMerge(Sorter::Run* merged_run) WARN_UNUSED_RESULT;
/// Called once there no more rows to be added to 'unsorted_run_'. Sorts
/// 'unsorted_run_' and appends it to the list of sorted runs.
- Status SortCurrentInputRun();
+ Status SortCurrentInputRun() WARN_UNUSED_RESULT;
/// Helper that cleans up all runs in the sorter.
void CleanupAllRuns();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/7a0ee685/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
index 5ace4f4..626b315 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
@@ -233,3 +233,20 @@ order by c_custkey
---- TYPES
bigint, bigint
====
+---- QUERY
+# IMPALA-5446: dropped status from Sorter::Reset() when sort cannot get reserved buffer.
+# This query is designed to allow the initial subplan iterations to succeed, but have
+# later iterations fail because the aggregation outside the subplan has accumulated all
+# the memory.
+set max_block_mgr_memory=100m;
+select c_custkey, c_name, c_address, c_phone, c_acctbal, c_mktsegment, c_comment,
+ o_orderdate, sum(o_totalprice), min(rnum)
+from customer c,
+ (select o_orderkey, o_totalprice, o_orderdate, row_number() over (order by o_orderdate desc) rnum
+ from c.c_orders) v
+group by 1, 2, 3, 4, 5, 6, 7, 8
+order by 9, 10 desc
+limit 10
+---- CATCH
+Memory limit exceeded: Query did not have enough memory to get the minimum required buffers in the block manager.
+====
[7/7] incubator-impala git commit: IMPALA-5263: test infra: support
CA bundles with secure clusters
Posted by ta...@apache.org.
IMPALA-5263: test infra: support CA bundles with secure clusters
This patch adds the command line option --ca_cert to the common test
infra CLI options for use alongside --use-ssl. This is useful when
testing against a secured Impala cluster in which the SSL certs are
self-signed. This will allow the SSL request to be validated. Using this
option will also suppress noisy console warnings like:
InsecureRequestWarning: Unverified HTTPS request is being made. Adding
certificate verification is strongly advised. See:
https://urllib3.readthedocs.org/en/latest/security.html
We also go further in this patch and use the warnings module to print
these SSL-related warnings once and only once, instead of all over the
place. In the case of the stress test, this greatly reduces the noise in
the console log.
Testing:
- quick concurrent_select.py calls with and without --ca_cert to observe
that connections still get made and the test runs smoothly. Some of
this testing occurred without warning suppression, so that I could be
sure the InsecureRequestWarnings were not occurring when using
--ca_cert anymore.
- ensured warnings are printed once, not multiple times
Change-Id: Ifb9e466e4b7cde704cdc4cf98159c068c0a400a9
Reviewed-on: http://gerrit.cloudera.org:8080/7152
Reviewed-by: David Knupp <dk...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/428b5a1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/428b5a1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/428b5a1b
Branch: refs/heads/master
Commit: 428b5a1bfe5e8a533db95c98f8ffbc1f825cdcef
Parents: 467ccd1
Author: Michael Brown <mi...@cloudera.com>
Authored: Sat Jun 10 16:35:00 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Jun 13 19:25:57 2017 +0000
----------------------------------------------------------------------
tests/comparison/cli_options.py | 30 ++++++++++++++++++++++++++++++
tests/comparison/cluster.py | 31 ++++++++++++++++++++++---------
tests/comparison/db_connection.py | 6 ++++--
3 files changed, 56 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/428b5a1b/tests/comparison/cli_options.py
----------------------------------------------------------------------
diff --git a/tests/comparison/cli_options.py b/tests/comparison/cli_options.py
index 91dd0ce..885ef84 100644
--- a/tests/comparison/cli_options.py
+++ b/tests/comparison/cli_options.py
@@ -21,7 +21,12 @@ import logging
import os
import sys
from getpass import getuser
+from requests.packages.urllib3.exceptions import (
+ InsecurePlatformWarning,
+ InsecureRequestWarning,
+ SecurityWarning)
from tempfile import gettempdir
+from warnings import filterwarnings
from tests.comparison import db_connection
from tests.comparison.cluster import (
@@ -172,6 +177,10 @@ def add_ssl_options(parser):
group.add_argument(
'--use-ssl', action='store_true', default=False,
help='Use SSL to connect')
+ group.add_argument(
+ '--ca_cert', default=None, metavar='CA cert path',
+ help='Path to optional CA certificate. This is needed to verify SSL requests if '
+ 'the Impala certificate is self-signed in a test environment.')
def create_cluster(args):
@@ -187,6 +196,27 @@ def create_cluster(args):
cluster.hadoop_user_name = args.hadoop_user_name
cluster.use_kerberos = getattr(args, 'use_kerberos', False)
cluster.use_ssl = getattr(args, 'use_ssl', False)
+ if cluster.use_ssl:
+ # Prevent excessive warning spam on the console.
+ #
+ # The first warning is related to certificates that do not comply with RFC 2818.
+ # https://github.com/shazow/urllib3/issues/497 . Permit one warning.
+ filterwarnings(
+ 'once',
+ 'Certificate has no `subjectAltName`',
+ SecurityWarning)
+ # Permit one warning with unverified HTTPS requests
+ filterwarnings(
+ 'once',
+ 'Unverified HTTPS request is being made',
+ InsecureRequestWarning)
+ # TODO: IMPALA-5264 to fix python environment to prevent InsecurePlatformWarning .
+ # Once we fix that we should remove this suppression.
+ filterwarnings(
+ 'once',
+ 'A true SSLContext object is not available',
+ InsecurePlatformWarning)
+ cluster.ca_cert = getattr(args, 'ca_cert', None)
return cluster
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/428b5a1b/tests/comparison/cluster.py
----------------------------------------------------------------------
diff --git a/tests/comparison/cluster.py b/tests/comparison/cluster.py
index e0328b0..761bc57 100644
--- a/tests/comparison/cluster.py
+++ b/tests/comparison/cluster.py
@@ -78,6 +78,7 @@ class Cluster(object):
self.hadoop_user_name = getuser()
self.use_kerberos = False
self.use_ssl = False
+ self.ca_cert = None
self._hdfs = None
self._yarn = None
@@ -491,9 +492,15 @@ class Hive(Service):
return self._warehouse_dir
def connect(self, db_name=None):
- conn = HiveConnection(host_name=self.hs2_host_name, port=self.hs2_port,
- user_name=self.cluster.hadoop_user_name, db_name=db_name,
- use_kerberos=self.cluster.use_kerberos, use_ssl=self.cluster.use_ssl)
+ conn = HiveConnection(
+ host_name=self.hs2_host_name,
+ port=self.hs2_port,
+ user_name=self.cluster.hadoop_user_name,
+ db_name=db_name,
+ use_kerberos=self.cluster.use_kerberos,
+ use_ssl=self.cluster.use_ssl,
+ ca_cert=self.cluster.ca_cert,
+ )
conn.cluster = self.cluster
return conn
@@ -525,9 +532,15 @@ class Impala(Service):
def connect(self, db_name=None, impalad=None):
if not impalad:
impalad = choice(self.impalads)
- conn = ImpalaConnection(host_name=impalad.host_name, port=impalad.hs2_port,
- user_name=self.cluster.hadoop_user_name, db_name=db_name,
- use_kerberos=self.cluster.use_kerberos, use_ssl=self.cluster.use_ssl)
+ conn = ImpalaConnection(
+ host_name=impalad.host_name,
+ port=impalad.hs2_port,
+ user_name=self.cluster.hadoop_user_name,
+ db_name=db_name,
+ use_kerberos=self.cluster.use_kerberos,
+ use_ssl=self.cluster.use_ssl,
+ ca_cert=self.cluster.ca_cert,
+ )
conn.cluster = self.cluster
return conn
@@ -774,9 +787,9 @@ class Impalad(object):
port=self.web_ui_port,
url=relative_url)
try:
- # verify=False is needed because of self-signed certifiates
- # TODO: support a CA bundle that users could point to instead
- resp = requests.get(url, params=params, timeout=timeout_secs, verify=False)
+ verify_ca = self.cluster.ca_cert if self.cluster.ca_cert is not None else False
+ resp = requests.get(url, params=params, timeout=timeout_secs,
+ verify=verify_ca)
except requests.exceptions.Timeout as e:
raise Timeout(underlying_exception=e)
resp.raise_for_status()
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/428b5a1b/tests/comparison/db_connection.py
----------------------------------------------------------------------
diff --git a/tests/comparison/db_connection.py b/tests/comparison/db_connection.py
index 6f9b7ce..71786b8 100644
--- a/tests/comparison/db_connection.py
+++ b/tests/comparison/db_connection.py
@@ -853,10 +853,11 @@ class ImpalaConnection(DbConnection):
_KERBEROS_SERVICE_NAME = 'impala'
_NON_KERBEROS_AUTH_MECH = 'NOSASL'
- def __init__(self, use_kerberos=False, use_ssl=False, **kwargs):
+ def __init__(self, use_kerberos=False, use_ssl=False, ca_cert=None, **kwargs):
self._use_kerberos = use_kerberos
self.cluster = None
self._use_ssl = use_ssl
+ self._ca_cert = ca_cert
DbConnection.__init__(self, **kwargs)
def clone(self, db_name):
@@ -887,7 +888,8 @@ class ImpalaConnection(DbConnection):
timeout=(60 * 60),
auth_mechanism=('GSSAPI' if self._use_kerberos else self._NON_KERBEROS_AUTH_MECH),
kerberos_service_name=self._KERBEROS_SERVICE_NAME,
- use_ssl=self._use_ssl)
+ use_ssl=self._use_ssl,
+ ca_cert=self._ca_cert)
class HiveCursor(ImpalaCursor):
[4/7] incubator-impala git commit: IMPALA-5487: Fix race in
RuntimeProfile::toThrift()
Posted by ta...@apache.org.
IMPALA-5487: Fix race in RuntimeProfile::toThrift()
node.num_children got initialized before children_ was locked. This
could lead to node.num_children < children_.size(), which made the node
tree in the resulting thrift profiles not deserialize properly.
To fix this, node.num_children needs to be initialized after children_
has been locked.
I tested this by running queries on a private cluster for a while,
making sure that the thrift profiles of running queries could be parsed
correctly.
Change-Id: I7fad4ee2eee1f73e387c1e90a3db265b19b3a0c6
Reviewed-on: http://gerrit.cloudera.org:8080/7154
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/58baca74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/58baca74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/58baca74
Branch: refs/heads/master
Commit: 58baca7455fb5ff153455fe7b53292ab5e431371
Parents: 9fd46f7
Author: Lars Volker <lv...@cloudera.com>
Authored: Sun Jun 11 22:01:34 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Jun 13 00:34:21 2017 +0000
----------------------------------------------------------------------
be/src/util/runtime-profile.cc | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/58baca74/be/src/util/runtime-profile.cc
----------------------------------------------------------------------
diff --git a/be/src/util/runtime-profile.cc b/be/src/util/runtime-profile.cc
index 9efe556..5ff9d28 100644
--- a/be/src/util/runtime-profile.cc
+++ b/be/src/util/runtime-profile.cc
@@ -761,13 +761,10 @@ void RuntimeProfile::ToThrift(TRuntimeProfileTree* tree) const {
}
void RuntimeProfile::ToThrift(vector<TRuntimeProfileNode>* nodes) const {
- nodes->reserve(nodes->size() + children_.size());
-
int index = nodes->size();
nodes->push_back(TRuntimeProfileNode());
TRuntimeProfileNode& node = (*nodes)[index];
node.name = name_;
- node.num_children = children_.size();
node.metadata = metadata_;
node.indent = true;
@@ -839,6 +836,7 @@ void RuntimeProfile::ToThrift(vector<TRuntimeProfileNode>* nodes) const {
{
lock_guard<SpinLock> l(children_lock_);
children = children_;
+ node.num_children = children_.size();
}
for (int i = 0; i < children.size(); ++i) {
int child_idx = nodes->size();
[6/7] incubator-impala git commit: IMPALA-5223: Add waiting for HBase
Zookeeper nodes to retry loop
Posted by ta...@apache.org.
IMPALA-5223: Add waiting for HBase Zookeeper nodes to retry loop
Occasionally we'd see HBase fail to startup properly on CentOS 7
clusters. The symptom was that HBase would not open the required nodes
in zookeeper, signaling its readiness.
As a workaround, this change includes waiting for the Zookeeper nodes
into the retry logic.
Change-Id: Id8dbdff4ad02cac1322e7d580e0a6971daf6ea28
Reviewed-on: http://gerrit.cloudera.org:8080/7159
Reviewed-by: Michael Brown <mi...@cloudera.com>
Reviewed-by: anujphadke <ap...@cloudera.com>
Reviewed-by: David Knupp <dk...@cloudera.com>
Tested-by: Lars Volker <lv...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/467ccd19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/467ccd19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/467ccd19
Branch: refs/heads/master
Commit: 467ccd19508eca0733cb061497a3c2ceca3ea849
Parents: 7a0ee68
Author: Lars Volker <lv...@cloudera.com>
Authored: Mon Jun 12 15:46:25 2017 -0700
Committer: Lars Volker <lv...@cloudera.com>
Committed: Tue Jun 13 05:57:49 2017 +0000
----------------------------------------------------------------------
testdata/bin/run-hbase.sh | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/467ccd19/testdata/bin/run-hbase.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/run-hbase.sh b/testdata/bin/run-hbase.sh
index 2a51105..f264b65 100755
--- a/testdata/bin/run-hbase.sh
+++ b/testdata/bin/run-hbase.sh
@@ -111,20 +111,27 @@ for ((i=1; i <= HBASE_START_RETRY_ATTEMPTS; ++i)); do
if ! ${HBASE_HOME}/bin/start-hbase.sh 2>&1 | tee -a ${HBASE_LOGDIR}/hbase-startup.out
then
echo "HBase Master startup failed"
- elif ! ${HBASE_HOME}/bin/local-regionservers.sh start 2 3 2>&1 | \
+ continue
+ fi
+ if ! ${HBASE_HOME}/bin/local-regionservers.sh start 2 3 2>&1 | \
tee -a ${HBASE_LOGDIR}/hbase-rs-startup.out
then
echo "HBase regionserver startup failed"
- else
- break
+ continue
+ fi
+ if ! ${CLUSTER_BIN}/check-hbase-nodes.py; then
+ echo "HBase nodes did not come online"
+ continue
fi
+ # If we made it to here, HBase started up correctly so we can stop the retry logic.
+ break
else
# In the last iteration, it's fine for errexit to do its thing.
${HBASE_HOME}/bin/start-hbase.sh 2>&1 | tee -a ${HBASE_LOGDIR}/hbase-startup.out
${HBASE_HOME}/bin/local-regionservers.sh start 2 3 2>&1 | \
tee -a ${HBASE_LOGDIR}/hbase-rs-startup.out
+ ${CLUSTER_BIN}/check-hbase-nodes.py
fi
done
-${CLUSTER_BIN}/check-hbase-nodes.py
echo "HBase startup scripts succeeded"
[2/7] incubator-impala git commit: IMPALA-5016: Simplify COALESCE()
in SimplifyConditionalsRule.
Posted by ta...@apache.org.
IMPALA-5016: Simplify COALESCE() in SimplifyConditionalsRule.
Simplify COALESCE by skipping leading nulls and applying the following
transformations:
COALESCE(null, a, b) -> COALESCE(a, b);
COALESCE(<literal>, a, b) -> <literal>, when literal is not NullLiteral;
COALESCE(<partition-slotref>, a, b) -> <partition-slotref>,
when the partition column does not contain NULL.
Testing:
added unit tests in ExprRewriteRulesTest
Change-Id: I0325a9e437261b15313bbdf2f22d83376a84b575
Reviewed-on: http://gerrit.cloudera.org:8080/7015
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/61696f3b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/61696f3b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/61696f3b
Branch: refs/heads/master
Commit: 61696f3ba81ba27d830bf262c6e56b0712f19e53
Parents: de9f523
Author: hzfengyu <hz...@corp.netease.com>
Authored: Sun May 28 23:58:18 2017 +0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Mon Jun 12 20:31:03 2017 +0000
----------------------------------------------------------------------
.../rewrite/SimplifyConditionalsRule.java | 94 +++++++++++++++++---
.../impala/analysis/ExprRewriteRulesTest.java | 49 +++++++++-
2 files changed, 126 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/61696f3b/fe/src/main/java/org/apache/impala/rewrite/SimplifyConditionalsRule.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/rewrite/SimplifyConditionalsRule.java b/fe/src/main/java/org/apache/impala/rewrite/SimplifyConditionalsRule.java
index 176b89e..287b4be 100644
--- a/fe/src/main/java/org/apache/impala/rewrite/SimplifyConditionalsRule.java
+++ b/fe/src/main/java/org/apache/impala/rewrite/SimplifyConditionalsRule.java
@@ -30,8 +30,13 @@ import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.FunctionCallExpr;
import org.apache.impala.analysis.FunctionName;
import org.apache.impala.analysis.NullLiteral;
+import org.apache.impala.analysis.SlotDescriptor;
+import org.apache.impala.analysis.SlotRef;
+import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.common.AnalysisException;
+
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
/***
* This rule simplifies conditional functions with constant conditions. It relies on
@@ -43,6 +48,7 @@ import com.google.common.base.Preconditions;
* id = 0 OR false -> id = 0
* false AND id = 1 -> false
* case when false then 0 when true then 1 end -> 1
+ * coalesce(1, 0) -> 1
*/
public class SimplifyConditionalsRule implements ExprRewriteRule {
public static ExprRewriteRule INSTANCE = new SimplifyConditionalsRule();
@@ -76,24 +82,84 @@ public class SimplifyConditionalsRule implements ExprRewriteRule {
* Simplifies IF by returning the corresponding child if the condition has a constant
* TRUE, FALSE, or NULL (equivalent to FALSE) value.
*/
+ private Expr simplifyIfFunctionCallExpr(FunctionCallExpr expr) {
+ Preconditions.checkState(expr.getChildren().size() == 3);
+ if (expr.getChild(0) instanceof BoolLiteral) {
+ if (((BoolLiteral) expr.getChild(0)).getValue()) {
+ // IF(TRUE)
+ return expr.getChild(1);
+ } else {
+ // IF(FALSE)
+ return expr.getChild(2);
+ }
+ } else if (expr.getChild(0) instanceof NullLiteral) {
+ // IF(NULL)
+ return expr.getChild(2);
+ }
+ return expr;
+ }
+
+ /**
+ * Simplify COALESCE by skipping leading nulls and applying the following transformations:
+ * COALESCE(null, a, b) -> COALESCE(a, b);
+ * COALESCE(<literal>, a, b) -> <literal>, when literal is not NullLiteral;
+ * COALESCE(<partition-slotref>, a, b) -> <partition-slotref>,
+ * when the partition column does not contain NULL.
+ */
+ private Expr simplifyCoalesceFunctionCallExpr(FunctionCallExpr expr) {
+ int numChildren = expr.getChildren().size();
+ Expr result = NullLiteral.create(expr.getType());
+ for (int i = 0; i < numChildren; ++i) {
+ Expr childExpr = expr.getChildren().get(i);
+ // Skip leading nulls.
+ if (childExpr.isNullLiteral()) continue;
+ if ((i == numChildren - 1) || canSimplifyCoalesceUsingChild(childExpr)) {
+ result = childExpr;
+ } else if (i == 0) {
+ result = expr;
+ } else {
+ List<Expr> newChildren = Lists.newArrayList(expr.getChildren().subList(i, numChildren));
+ result = new FunctionCallExpr(expr.getFnName(), newChildren);
+ }
+ break;
+ }
+ return result;
+ }
+
+ /**
+ * Checks if the given child expr is nullable. Returns true if one of the following holds:
+ * child is a non-NULL literal;
+ * child is a possibly cast SlotRef against a non-nullable slot;
+ * child is a possible cast SlotRef against a partition column that does not contain NULL.
+ */
+ private boolean canSimplifyCoalesceUsingChild(Expr child) {
+ if (child.isLiteral() && !child.isNullLiteral()) return true;
+
+ SlotRef slotRef = child.unwrapSlotRef(false);
+ if (slotRef == null) return false;
+ SlotDescriptor slotDesc = slotRef.getDesc();
+ if (!slotDesc.getIsNullable()) return true;
+ // Check partition column using partition metadata.
+ if (slotDesc.getParent().getTable() instanceof HdfsTable
+ && slotDesc.getColumn() != null
+ && slotDesc.getParent().getTable().isClusteringColumn(slotDesc.getColumn())) {
+ HdfsTable table = (HdfsTable) slotDesc.getParent().getTable();
+ // Return true if the partition column does not have a NULL value.
+ if (table.getNullPartitionIds(slotDesc.getColumn().getPosition()).isEmpty()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private Expr simplifyFunctionCallExpr(FunctionCallExpr expr) {
FunctionName fnName = expr.getFnName();
// TODO: Add the other conditional functions, eg. ifnull, istrue, etc.
if (fnName.getFunction().equals("if")) {
- Preconditions.checkState(expr.getChildren().size() == 3);
- if (expr.getChild(0) instanceof BoolLiteral) {
- if (((BoolLiteral) expr.getChild(0)).getValue()) {
- // IF(TRUE)
- return expr.getChild(1);
- } else {
- // IF(FALSE)
- return expr.getChild(2);
- }
- } else if (expr.getChild(0) instanceof NullLiteral) {
- // IF(NULL)
- return expr.getChild(2);
- }
+ return simplifyIfFunctionCallExpr(expr);
+ } else if (fnName.getFunction().equals("coalesce")) {
+ return simplifyCoalesceFunctionCallExpr(expr);
}
return expr;
}
@@ -209,4 +275,4 @@ public class SimplifyConditionalsRule implements ExprRewriteRule {
}
return new CaseExpr(caseExpr, newWhenClauses, elseExpr);
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/61696f3b/fe/src/test/java/org/apache/impala/analysis/ExprRewriteRulesTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ExprRewriteRulesTest.java b/fe/src/test/java/org/apache/impala/analysis/ExprRewriteRulesTest.java
index 3ee4141..9650f76 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ExprRewriteRulesTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ExprRewriteRulesTest.java
@@ -42,12 +42,22 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
public Expr RewritesOk(String expr, ExprRewriteRule rule, String expectedExpr)
throws AnalysisException {
- return RewritesOk(expr, Lists.newArrayList(rule), expectedExpr);
+ return RewritesOk("functional.alltypessmall", expr, rule, expectedExpr);
+ }
+
+ public Expr RewritesOk(String tableName, String expr, ExprRewriteRule rule, String expectedExpr)
+ throws AnalysisException {
+ return RewritesOk(tableName, expr, Lists.newArrayList(rule), expectedExpr);
}
public Expr RewritesOk(String expr, List<ExprRewriteRule> rules, String expectedExpr)
throws AnalysisException {
- String stmtStr = "select " + expr + " from functional.alltypessmall";
+ return RewritesOk("functional.alltypessmall", expr, rules, expectedExpr);
+ }
+
+ public Expr RewritesOk(String tableName, String expr, List<ExprRewriteRule> rules,
+ String expectedExpr) throws AnalysisException {
+ String stmtStr = "select " + expr + " from " + tableName;
SelectStmt stmt = (SelectStmt) ParsesOk(stmtStr);
Analyzer analyzer = createAnalyzer(Catalog.DEFAULT_DB);
stmt.analyze(analyzer);
@@ -333,6 +343,39 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
RewritesOk("case when true then 0 when false then sum(id) end", rule, null);
RewritesOk(
"case when true then count(id) when false then sum(id) end", rule, "count(id)");
+
+ // IMPALA-5016: Simplify COALESCE function
+ // Test skipping leading nulls.
+ RewritesOk("coalesce(null, id, year)", rule, "coalesce(id, year)");
+ RewritesOk("coalesce(null, 1, id)", rule, "1");
+ RewritesOk("coalesce(null, null, id)", rule, "id");
+ // If the leading parameter is a non-NULL constant, rewrite to that constant.
+ RewritesOk("coalesce(1, id, year)", rule, "1");
+ // If COALESCE has only one parameter, rewrite to the parameter.
+ RewritesOk("coalesce(id)", rule, "id");
+ // If all parameters are NULL, rewrite to NULL.
+ RewritesOk("coalesce(null, null)", rule, "NULL");
+ // Do not rewrite non-literal constant exprs, rely on constant folding.
+ RewritesOk("coalesce(null is null, id)", rule, null);
+ RewritesOk("coalesce(10 + null, id)", rule, null);
+ // Combine COALESCE rule with FoldConstantsRule.
+ RewritesOk("coalesce(1 + 2, id, year)", rules, "3");
+ RewritesOk("coalesce(null is null, bool_col)", rules, "TRUE");
+ RewritesOk("coalesce(10 + null, id, year)", rules, "coalesce(id, year)");
+ // If the leading parameter is partition column, try to rewrite with partition metadata.
+ RewritesOk("coalesce(year, id)", rule, "year");
+ RewritesOk("coalesce(year, bigint_col)", rule, "year");
+ RewritesOk("coalesce(cast(year as string), string_col)", rule, "CAST(year AS STRING)");
+ RewritesOk("coalesce(id, year)", rule, null);
+ RewritesOk("coalesce(null, year, id)", rule, "year");
+ // If the leading partition column has NULL value, do not rewrite.
+ RewritesOk("functional.alltypesagg", "coalesce(year, id)", rule, "year");
+ RewritesOk("functional.alltypesagg", "coalesce(day, id)", rule, null);
+ // If the leading column is not nullable, rewrite to the column.
+ RewritesOk("functional_kudu.alltypessmall", "coalesce(id, year)", rule, "id");
+ RewritesOk("functional_kudu.alltypessmall", "coalesce(cast(id as string), string_col)", rule,
+ "CAST(id AS STRING)");
+ RewritesOk("functional_kudu.alltypessmall", "coalesce(null, id, year)", rule, "id");
}
@Test
@@ -364,4 +407,4 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
RewritesOk("int_col = tinyint_col", rule, null);
RewritesOk("tinyint_col = int_col", rule, null);
}
-}
+}
\ No newline at end of file