You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by kw...@apache.org on 2016/11/23 08:25:22 UTC
[1/6] incubator-impala git commit: IMPALA-4516: Don't hold process
wide lock connection_to_sessions_map_lock_ while cancelling queries
Repository: incubator-impala
Updated Branches:
refs/heads/master b7eeb8bf8 -> 4db330e69
IMPALA-4516: Don't hold process wide lock connection_to_sessions_map_lock_ while cancelling queries
We hold the connection_to_sessions_map_lock_ while closing multiple
sessions, which could map to a large number of queries, which means an
even larger number of fragments. We hold this process wide lock and a
series of other locks while sending cancel RPCs to all the fragments
that fall under the above mentioned category.
This could slow down the responsiveness to the client by the daemon.
Moreover, holding the lock is unnecessary and we can do without it.
This patch fixes this path by dropping the lock before we call
CloseSessionInternal().
Change-Id: I9fe37955027ad9fec3fdbbbcb199245c79bcac71
Reviewed-on: http://gerrit.cloudera.org:8080/5173
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/4821ea6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/4821ea6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/4821ea6c
Branch: refs/heads/master
Commit: 4821ea6c468d3d4ae37d47b21aafaee553fb0549
Parents: b7eeb8b
Author: Sailesh Mukil <sa...@cloudera.com>
Authored: Mon Nov 21 12:51:43 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 22 22:26:58 2016 +0000
----------------------------------------------------------------------
be/src/service/impala-server.cc | 24 ++++++++++++++++--------
be/src/service/impala-server.h | 18 +++++++++---------
2 files changed, 25 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4821ea6c/be/src/service/impala-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index 49e81c8..5e08224 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -1654,24 +1654,32 @@ void ImpalaServer::ConnectionStart(
void ImpalaServer::ConnectionEnd(
const ThriftServer::ConnectionContext& connection_context) {
- unique_lock<mutex> l(connection_to_sessions_map_lock_);
- ConnectionToSessionMap::iterator it =
- connection_to_sessions_map_.find(connection_context.connection_id);
- // Not every connection must have an associated session
- if (it == connection_to_sessions_map_.end()) return;
+ vector<TUniqueId> sessions_to_close;
+ {
+ unique_lock<mutex> l(connection_to_sessions_map_lock_);
+ ConnectionToSessionMap::iterator it =
+ connection_to_sessions_map_.find(connection_context.connection_id);
+
+ // Not every connection must have an associated session
+ if (it == connection_to_sessions_map_.end()) return;
+
+ // We don't expect a large number of sessions per connection, so we copy it, so that
+ // we can drop the map lock early.
+ sessions_to_close = it->second;
+ connection_to_sessions_map_.erase(it);
+ }
LOG(INFO) << "Connection from client " << connection_context.network_address
- << " closed, closing " << it->second.size() << " associated session(s)";
+ << " closed, closing " << sessions_to_close.size() << " associated session(s)";
- for (const TUniqueId& session_id: it->second) {
+ for (const TUniqueId& session_id: sessions_to_close) {
Status status = CloseSessionInternal(session_id, true);
if (!status.ok()) {
LOG(WARNING) << "Error closing session " << session_id << ": "
<< status.GetDetail();
}
}
- connection_to_sessions_map_.erase(it);
}
void ImpalaServer::RegisterSessionTimeout(int32_t session_timeout) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4821ea6c/be/src/service/impala-server.h
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h
index 9d14ecd..d87af39 100644
--- a/be/src/service/impala-server.h
+++ b/be/src/service/impala-server.h
@@ -78,15 +78,14 @@ class TGetExecSummaryReq;
/// This class is partially thread-safe. To ensure freedom from deadlock, if multiple
/// locks are acquired, lower-numbered locks must be acquired before higher-numbered
/// locks:
-/// 1. connection_to_sessions_map_lock_
-/// 2. session_state_map_lock_
-/// 3. SessionState::lock
-/// 4. query_expiration_lock_
-/// 5. query_exec_state_map_lock_
-/// 6. QueryExecState::fetch_rows_lock
-/// 7. QueryExecState::lock
-/// 8. QueryExecState::expiration_data_lock_
-/// 9. Coordinator::exec_summary_lock
+/// 1. session_state_map_lock_
+/// 2. SessionState::lock
+/// 3. query_expiration_lock_
+/// 4. query_exec_state_map_lock_
+/// 5. QueryExecState::fetch_rows_lock
+/// 6. QueryExecState::lock
+/// 7. QueryExecState::expiration_data_lock_
+/// 8. Coordinator::exec_summary_lock
///
/// Coordinator::lock_ should not be acquired at the same time as the
/// ImpalaServer/SessionState/QueryExecState locks. Aside from
@@ -99,6 +98,7 @@ class TGetExecSummaryReq;
/// * query_locations_lock_
/// * uuid_lock_
/// * catalog_version_lock_
+/// * connection_to_sessions_map_lock_
///
/// TODO: The state of a running query is currently not cleaned up if the
/// query doesn't experience any errors at runtime and close() doesn't get called.
[4/6] incubator-impala git commit: IMPALA-4510: Selectively filter
args for metric verification tests
Posted by kw...@apache.org.
IMPALA-4510: Selectively filter args for metric verification tests
run-tests.py is a wrapper around impala-py.test. It abstracts away
the need to invoke separate runs for serial tests, parallel tests,
and metric verification tests.
Because it's possible for a user to specify certain test suites,
or even specific tests, on the command line when calling
run-tests.py, it had been necessary to override the command line
args when it came time to run the metric verification tests --
otherwise those other tests/suites would be rerun. Before this
patch, we had simply been stripping away all command line args.
However, that blanket approach causes problems when running tests
against a remote cluster, because we need to retain those command
line args that pertain to the remote cluster.
This patch selectively prunes unwanted command line args for the
last metric verification test stage, keeping the ones that we
need, and also adds extensive documentation for explaining why we
have to go through this fairly odd and elaborate step.
This patch was tested by running a sample test suite locally,
and against a remote cluster. Previously, the metric verification
stage had been failing for remote cluster tests (since they were
defaulting to localhost for services that were only available
remotely.) With the patch, the remote verfification tests were
passing.
Also, while I'm here, add a small change that exits immediately
if the user calls for --help. Before this, we actually still ran
the tests.
Change-Id: I069172f44c1307d55f85779cdb01fecc0ba1799e
Reviewed-on: http://gerrit.cloudera.org:8080/5135
Reviewed-by: Michael Brown <mi...@cloudera.com>
Reviewed-by: Ishaan Joshi <is...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/696fb68e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/696fb68e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/696fb68e
Branch: refs/heads/master
Commit: 696fb68e58897868d6c6984f81c5823937ab4069
Parents: bb1c633
Author: David Knupp <dk...@cloudera.com>
Authored: Thu Nov 17 20:24:07 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Wed Nov 23 05:46:18 2016 +0000
----------------------------------------------------------------------
tests/run-tests.py | 91 +++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 77 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/696fb68e/tests/run-tests.py
----------------------------------------------------------------------
diff --git a/tests/run-tests.py b/tests/run-tests.py
index c33bfd3..de32741 100755
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -20,6 +20,7 @@
# Runs the Impala query tests, first executing the tests that cannot be run in parallel
# and then executing the remaining tests in parallel. All additional command line options
# are passed to py.test.
+import itertools
import multiprocessing
import os
import pytest
@@ -54,6 +55,7 @@ NUM_STRESS_CLIENTS = min(multiprocessing.cpu_count() * 4, 64)
if 'NUM_STRESS_CLIENTS' in os.environ:
NUM_STRESS_CLIENTS = int(os.environ['NUM_STRESS_CLIENTS'])
+
class TestExecutor:
def __init__(self, exit_on_error=True):
self._exit_on_error = exit_on_error
@@ -65,30 +67,94 @@ class TestExecutor:
sys.exit(exit_code)
self.tests_failed = exit_code != 0 or self.tests_failed
-def build_test_args(log_base_name, valid_dirs, include_cmdline_args=True):
- logging_args = LOGGING_ARGS % {'result_dir': TEST_RESULT_DIR, 'log_name': log_base_name}
- args = '%s %s' % (build_ignore_dir_arg_list(valid_dirs=valid_dirs), logging_args)
- if include_cmdline_args:
- # sys.argv[0] is always the script name, so exclude it
- return '%s %s' % (args, ' '.join(sys.argv[1:]))
- return args
+def build_test_args(log_base_name, valid_dirs):
+ """
+ Modify and return the command line arguments that will be passed to py.test.
+
+ Args:
+ log_base_name: the base name for the log file to write
+ valid_dirs: a white list of sub-directories with desired tests (i.e, those
+ that will not get flagged with --ignore before py.test is called.)
+
+ Return:
+ a modified command line for py.test
+
+ For most test stages (e.g., serial, parallel), we augment the given command
+ line arguments with a list of directories to ignore. However, when running the
+ metric verification tests at the end of the test run:
+
+ - verifiers.test_verify_metrics.TestValidateMetrics.test_metrics_are_zero
+ - verifiers.test_verify_metrics.TestValidateMetrics.test_num_unused_buffers
+
+ then we instead need to filter out args that specifiy other tests (otherwise,
+ they will be run again), but still retain the basic config args.
+ """
+ logging_args = LOGGING_ARGS % {'result_dir': TEST_RESULT_DIR,
+ 'log_name': log_base_name}
+
+ # The raw command line arguments need to be modified because of the way our
+ # repo is organized. We have several non-test directories and files in our
+ # tests/ path, which causes auto-discovery problems for pytest -- i.e., pytest
+ # will futiley try to execute them as tests, resulting in misleading failures.
+ # (There is a JIRA filed to restructure this: IMPALA-4417.)
+ #
+ # e.g. --ignore="comparison" --ignore="util" --ignore=etc...
+ ignored_dirs = build_ignore_dir_arg_list(valid_dirs=valid_dirs)
+
+ if valid_dirs != ['verifiers']:
+ # This isn't the metrics verification stage yet, so after determining the
+ # logging params and which sub-directories within tests/ to ignore, just tack
+ # on any other args from sys.argv -- excluding sys.argv[0], which of course
+ # is the script name
+ test_args = '%s %s %s' % (ignored_dirs, logging_args, ' '.join(sys.argv[1:]))
+ else:
+ # When filtering, we need to account for the fact that '--foo bar' and
+ # '--foo=bar' might be supplied by the user, as well as random options. E.g.,
+ # if the user specified the following on the command line:
+ #
+ # 'run-tests.py --arg1 value1 --random_opt --arg2=value2'
+ #
+ # we want an iterable that, if unpacked as a list, would look like:
+ #
+ # [arg1, value1, random_opt, arg2, value2]
+ #
+ raw_args = itertools.chain(*[arg.split('=') for arg in sys.argv[1:]])
+ kept_args = []
+
+ for arg in raw_args:
+ try:
+ pytest.config.getvalue(arg.strip('-')) # Raises ValueError if invalid arg
+ kept_args += [arg, str(raw_args.next())]
+ except ValueError:
+ # For any arg that's not a required pytest config arg, we can filter it out
+ continue
+ test_args = '%s %s %s' % (ignored_dirs, logging_args, ' '.join(kept_args))
+
+ return test_args
+
def build_ignore_dir_arg_list(valid_dirs):
""" Builds a list of directories to ignore """
subdirs = [subdir for subdir in os.listdir(TEST_DIR) if os.path.isdir(subdir)]
return ' '.join(['--ignore="%s"' % d for d in set(subdirs) - set(valid_dirs)])
+
if __name__ == "__main__":
+ exit_on_error = '-x' in sys.argv or '--exitfirst' in sys.argv
+ test_executor = TestExecutor(exit_on_error=exit_on_error)
+
+ # If the user is just asking for --help, just print the help test and then exit.
+ if '-h' in sys.argv[1:] or '--help' in sys.argv[1:]:
+ test_executor.run_tests(sys.argv[1:])
+ sys.exit(0)
+
os.chdir(TEST_DIR)
# Create the test result directory if it doesn't already exist.
if not os.path.exists(TEST_RESULT_DIR):
os.makedirs(TEST_RESULT_DIR)
- exit_on_error = '-x' in sys.argv or '--exitfirst' in sys.argv
- test_executor = TestExecutor(exit_on_error=exit_on_error)
-
# First run query tests that need to be executed serially
args = '-m "execute_serially" %s' % build_test_args('serial', VALID_TEST_DIRS)
test_executor.run_tests(args)
@@ -104,10 +170,7 @@ if __name__ == "__main__":
test_executor.run_tests(args)
# Finally, validate impalad/statestored metrics.
- # Do not include any command-line arguments when invoking verifiers because it
- # can lead to tests being run multiple times should someone pass in a .py file.
- args = build_test_args(log_base_name='verify-metrics', valid_dirs=['verifiers'],
- include_cmdline_args=False)
+ args = build_test_args(log_base_name='verify-metrics', valid_dirs=['verifiers'])
args += ' verifiers/test_verify_metrics.py'
test_executor.run_tests(args)
[6/6] incubator-impala git commit: IMPALA-4397,
IMPALA-3259: reduce codegen time and memory
Posted by kw...@apache.org.
IMPALA-4397,IMPALA-3259: reduce codegen time and memory
A handful of fixes to codegen memory usage:
* Delete the IR module when we're done with it (it can be fairly large)
* Track the compiled code size (typically not that large, but it can add
up if there are many fragments).
* Estimate optimisation memory requirements and track it in the memory
tracker. This is very crude but much better than not tracking it.
A handful of fixes to improve codegen time/cost, particularly targeted
at compute stats workloads:
* Avoid over-inlining when there are many aggregate functions,
conjuncts, etc by adding "NoInline" attributes.
* Don't codegen non-grouping merge aggregations. They will only process
one row per Impala daemon, so codegen is not worth it.
* Make the Hll algorithm more efficient by specialising the hash function
based on decimal width.
Limitations:
* This doesn't tackle over-inlining of large expr trees, but a similar
approach will be used there in a follow-on patch.
Perf:
Compute stats on functional_parquet.widetable_1000_cols goes from 1min+
of codegen to ~ 5s codegen on my machine. Local perf runs of tpc-h
and targeted perf showed no regressions and some moderate improvements
(1-2%).
Also did an experiment to understand the perf consequences of disabling
inlining. I manually set CODEGEN_INLINE_EXPRS_THRESHOLD to 0, and ran:
drop stats tpch_20_parquet.lineitem
compute stats tpch_20_parquet.lineitem;
There was no difference in time spent in the agg node: 30.7s with
inlining, 30.5s without.
Change-Id: Id10015b49da182cb181a653ac8464b4a18b71091
Reviewed-on: http://gerrit.cloudera.org:8080/4956
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Reviewed-by: Marcel Kornacker <ma...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/4db330e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/4db330e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/4db330e6
Branch: refs/heads/master
Commit: 4db330e69a2dbb4a23f46e34b484da0d6b9ef29b
Parents: 696fb68
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Tue Oct 4 17:09:17 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Wed Nov 23 08:18:17 2016 +0000
----------------------------------------------------------------------
be/src/benchmarks/hash-benchmark.cc | 2 +-
be/src/codegen/llvm-codegen-test.cc | 40 +-
be/src/codegen/llvm-codegen.cc | 114 +-
be/src/codegen/llvm-codegen.h | 84 +-
be/src/codegen/mcjit-mem-mgr.h | 38 +-
be/src/exec/aggregation-node.cc | 4 +-
be/src/exec/exchange-node.cc | 4 +-
be/src/exec/exec-node.cc | 17 +-
be/src/exec/exec-node.h | 7 +
be/src/exec/hash-join-node.cc | 17 +-
be/src/exec/hash-table.cc | 41 +-
be/src/exec/hdfs-scan-node-base.cc | 5 +-
be/src/exec/hdfs-scanner.cc | 9 +
be/src/exec/partitioned-aggregation-node.cc | 32 +-
be/src/exec/partitioned-aggregation-node.h | 2 +-
be/src/exec/partitioned-hash-join-node.cc | 11 +-
be/src/exec/sort-node.cc | 9 +-
be/src/exec/topn-node.cc | 4 +-
be/src/exprs/aggregate-functions-ir.cc | 22 +-
be/src/exprs/anyval-util.h | 19 +-
be/src/exprs/expr-codegen-test.cc | 3 +-
be/src/exprs/expr.h | 1 +
be/src/runtime/lib-cache.cc | 4 +-
be/src/runtime/runtime-state.cc | 14 +-
common/thrift/PlanNodes.thrift | 36 +-
.../impala/planner/DistributedPlanner.java | 6 +
.../org/apache/impala/planner/PlanNode.java | 12 +-
.../queries/QueryTest/compute-stats.test | 1016 ++++++++++++++++++
tests/query_test/test_aggregation.py | 51 +-
29 files changed, 1479 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/benchmarks/hash-benchmark.cc
----------------------------------------------------------------------
diff --git a/be/src/benchmarks/hash-benchmark.cc b/be/src/benchmarks/hash-benchmark.cc
index 4f6d7f1..125310b 100644
--- a/be/src/benchmarks/hash-benchmark.cc
+++ b/be/src/benchmarks/hash-benchmark.cc
@@ -429,7 +429,7 @@ int main(int argc, char **argv) {
Status status;
scoped_ptr<LlvmCodeGen> codegen;
- status = LlvmCodeGen::CreateImpalaCodegen(&obj_pool, "test", &codegen);
+ status = LlvmCodeGen::CreateImpalaCodegen(&obj_pool, NULL, "test", &codegen);
if (!status.ok()) {
cout << "Could not start codegen.";
return -1;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/codegen/llvm-codegen-test.cc
----------------------------------------------------------------------
diff --git a/be/src/codegen/llvm-codegen-test.cc b/be/src/codegen/llvm-codegen-test.cc
index 178baa2..1238dde 100644
--- a/be/src/codegen/llvm-codegen-test.cc
+++ b/be/src/codegen/llvm-codegen-test.cc
@@ -42,9 +42,9 @@ class LlvmCodeGenTest : public testing:: Test {
ObjectPool pool;
Status status;
for (int i = 0; i < 10; ++i) {
- LlvmCodeGen object1(&pool, "Test");
- LlvmCodeGen object2(&pool, "Test");
- LlvmCodeGen object3(&pool, "Test");
+ LlvmCodeGen object1(&pool, NULL, "Test");
+ LlvmCodeGen object2(&pool, NULL, "Test");
+ LlvmCodeGen object3(&pool, NULL, "Test");
ASSERT_OK(object1.Init(unique_ptr<Module>(new Module("Test", object1.context()))));
ASSERT_OK(object2.Init(unique_ptr<Module>(new Module("Test", object2.context()))));
@@ -53,16 +53,16 @@ class LlvmCodeGenTest : public testing:: Test {
}
// Wrapper to call private test-only methods on LlvmCodeGen object
- static Status CreateFromFile(ObjectPool* pool, const string& filename,
- scoped_ptr<LlvmCodeGen>* codegen) {
- return LlvmCodeGen::CreateFromFile(pool, filename, "test", codegen);
+ static Status CreateFromFile(
+ ObjectPool* pool, const string& filename, scoped_ptr<LlvmCodeGen>* codegen) {
+ return LlvmCodeGen::CreateFromFile(pool, NULL, filename, "test", codegen);
}
static LlvmCodeGen* CreateCodegen(ObjectPool* pool) {
- LlvmCodeGen* codegen = pool->Add(new LlvmCodeGen(pool, "Test"));
+ LlvmCodeGen* codegen = pool->Add(new LlvmCodeGen(pool, NULL, "Test"));
if (codegen != NULL) {
- Status status = codegen->Init(
- unique_ptr<Module>(new Module("Test", codegen->context())));
+ Status status =
+ codegen->Init(unique_ptr<Module>(new Module("Test", codegen->context())));
if (!status.ok()) return NULL;
}
return codegen;
@@ -82,10 +82,7 @@ class LlvmCodeGenTest : public testing:: Test {
return codegen->VerifyFunction(fn);
}
- static Status FinalizeModule(LlvmCodeGen* codegen) {
- return codegen->FinalizeModule();
- }
-
+ static Status FinalizeModule(LlvmCodeGen* codegen) { return codegen->FinalizeModule(); }
};
// Simple test to just make and destroy llvmcodegen objects. LLVM
@@ -109,7 +106,8 @@ TEST_F(LlvmCodeGenTest, BadIRFile) {
ObjectPool pool;
string module_file = "NonExistentFile.ir";
scoped_ptr<LlvmCodeGen> codegen;
- EXPECT_FALSE(LlvmCodeGenTest::CreateFromFile(&pool, module_file.c_str(), &codegen).ok());
+ EXPECT_FALSE(
+ LlvmCodeGenTest::CreateFromFile(&pool, module_file.c_str(), &codegen).ok());
}
// IR for the generated linner loop
@@ -289,7 +287,7 @@ TEST_F(LlvmCodeGenTest, StringValue) {
ObjectPool pool;
scoped_ptr<LlvmCodeGen> codegen;
- ASSERT_OK(LlvmCodeGen::CreateImpalaCodegen(&pool, "test", &codegen));
+ ASSERT_OK(LlvmCodeGen::CreateImpalaCodegen(&pool, NULL, "test", &codegen));
EXPECT_TRUE(codegen.get() != NULL);
string str("Test");
@@ -332,7 +330,7 @@ TEST_F(LlvmCodeGenTest, MemcpyTest) {
ObjectPool pool;
scoped_ptr<LlvmCodeGen> codegen;
- ASSERT_OK(LlvmCodeGen::CreateImpalaCodegen(&pool, "test", &codegen));
+ ASSERT_OK(LlvmCodeGen::CreateImpalaCodegen(&pool, NULL, "test", &codegen));
ASSERT_TRUE(codegen.get() != NULL);
LlvmCodeGen::FnPrototype prototype(codegen.get(), "MemcpyTest", codegen->void_type());
@@ -379,13 +377,13 @@ TEST_F(LlvmCodeGenTest, HashTest) {
// Loop to test both the sse4 on/off paths
for (int i = 0; i < 2; ++i) {
scoped_ptr<LlvmCodeGen> codegen;
- ASSERT_OK(LlvmCodeGen::CreateImpalaCodegen(&pool, "test", &codegen));
+ ASSERT_OK(LlvmCodeGen::CreateImpalaCodegen(&pool, NULL, "test", &codegen));
ASSERT_TRUE(codegen.get() != NULL);
- Value* llvm_data1 = codegen->CastPtrToLlvmPtr(codegen->ptr_type(),
- const_cast<char*>(data1));
- Value* llvm_data2 = codegen->CastPtrToLlvmPtr(codegen->ptr_type(),
- const_cast<char*>(data2));
+ Value* llvm_data1 =
+ codegen->CastPtrToLlvmPtr(codegen->ptr_type(), const_cast<char*>(data1));
+ Value* llvm_data2 =
+ codegen->CastPtrToLlvmPtr(codegen->ptr_type(), const_cast<char*>(data2));
Value* llvm_len1 = codegen->GetIntConstant(TYPE_INT, strlen(data1));
Value* llvm_len2 = codegen->GetIntConstant(TYPE_INT, strlen(data2));
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/codegen/llvm-codegen.cc
----------------------------------------------------------------------
diff --git a/be/src/codegen/llvm-codegen.cc b/be/src/codegen/llvm-codegen.cc
index f6e7e97..596f48a 100644
--- a/be/src/codegen/llvm-codegen.cc
+++ b/be/src/codegen/llvm-codegen.cc
@@ -25,9 +25,9 @@
#include <gutil/strings/substitute.h>
#include <llvm/ADT/Triple.h>
-#include <llvm/Analysis/TargetTransformInfo.h>
#include <llvm/Analysis/InstructionSimplify.h>
#include <llvm/Analysis/Passes.h>
+#include <llvm/Analysis/TargetTransformInfo.h>
#include <llvm/Bitcode/ReaderWriter.h>
#include <llvm/ExecutionEngine/ExecutionEngine.h>
#include <llvm/ExecutionEngine/MCJIT.h>
@@ -52,17 +52,18 @@
#include <llvm/Transforms/Utils/BasicBlockUtils.h>
#include <llvm/Transforms/Utils/Cloning.h>
-#include "common/logging.h"
#include "codegen/codegen-anyval.h"
#include "codegen/codegen-symbol-emitter.h"
#include "codegen/impala-ir-data.h"
#include "codegen/instruction-counter.h"
#include "codegen/mcjit-mem-mgr.h"
+#include "common/logging.h"
#include "impala-ir/impala-ir-names.h"
#include "runtime/descriptors.h"
#include "runtime/hdfs-fs-cache.h"
#include "runtime/lib-cache.h"
#include "runtime/mem-pool.h"
+#include "runtime/mem-tracker.h"
#include "runtime/string-value.h"
#include "runtime/timestamp-value.h"
#include "util/cpu-info.h"
@@ -76,7 +77,7 @@
using namespace llvm;
using namespace strings;
using std::fstream;
-using std::unique_ptr;
+using std::move;
DEFINE_bool(print_llvm_ir_instruction_count, false,
"if true, prints the instruction counts of all JIT'd functions");
@@ -200,7 +201,7 @@ void LlvmCodeGen::InitializeLlvm(bool load_backend) {
ObjectPool init_pool;
scoped_ptr<LlvmCodeGen> init_codegen;
- Status status = LlvmCodeGen::CreateFromMemory(&init_pool, "init", &init_codegen);
+ Status status = LlvmCodeGen::CreateFromMemory(&init_pool, NULL, "init", &init_codegen);
ParseGVForFunctions(init_codegen->module_, &gv_ref_ir_fns_);
// Validate the module by verifying that functions for all IRFunction::Type
@@ -213,16 +214,18 @@ void LlvmCodeGen::InitializeLlvm(bool load_backend) {
}
}
-LlvmCodeGen::LlvmCodeGen(ObjectPool* pool, const string& id) :
- id_(id),
- profile_(pool, "CodeGen"),
- optimizations_enabled_(false),
- is_corrupt_(false),
- is_compiled_(false),
- context_(new llvm::LLVMContext()),
- module_(NULL),
- loaded_functions_(IRFunction::FN_END, NULL) {
-
+LlvmCodeGen::LlvmCodeGen(
+ ObjectPool* pool, MemTracker* parent_mem_tracker, const string& id)
+ : id_(id),
+ profile_(pool, "CodeGen"),
+ mem_tracker_(new MemTracker(&profile_, -1, "CodeGen", parent_mem_tracker)),
+ optimizations_enabled_(false),
+ is_corrupt_(false),
+ is_compiled_(false),
+ context_(new llvm::LLVMContext()),
+ module_(NULL),
+ memory_manager_(NULL),
+ loaded_functions_(IRFunction::FN_END, NULL) {
DCHECK(llvm_initialized_) << "Must call LlvmCodeGen::InitializeLlvm first.";
load_module_timer_ = ADD_TIMER(&profile_, "LoadTime");
@@ -235,9 +238,9 @@ LlvmCodeGen::LlvmCodeGen(ObjectPool* pool, const string& id) :
num_instructions_ = ADD_COUNTER(&profile_, "NumInstructions", TUnit::UNIT);
}
-Status LlvmCodeGen::CreateFromFile(ObjectPool* pool,
+Status LlvmCodeGen::CreateFromFile(ObjectPool* pool, MemTracker* parent_mem_tracker,
const string& file, const string& id, scoped_ptr<LlvmCodeGen>* codegen) {
- codegen->reset(new LlvmCodeGen(pool, id));
+ codegen->reset(new LlvmCodeGen(pool, parent_mem_tracker, id));
SCOPED_TIMER((*codegen)->profile_.total_time_counter());
unique_ptr<Module> loaded_module;
@@ -246,9 +249,9 @@ Status LlvmCodeGen::CreateFromFile(ObjectPool* pool,
return (*codegen)->Init(std::move(loaded_module));
}
-Status LlvmCodeGen::CreateFromMemory(ObjectPool* pool, const string& id,
- scoped_ptr<LlvmCodeGen>* codegen) {
- codegen->reset(new LlvmCodeGen(pool, id));
+Status LlvmCodeGen::CreateFromMemory(ObjectPool* pool, MemTracker* parent_mem_tracker,
+ const string& id, scoped_ptr<LlvmCodeGen>* codegen) {
+ codegen->reset(new LlvmCodeGen(pool, parent_mem_tracker, id));
SCOPED_TIMER((*codegen)->profile_.total_time_counter());
// Select the appropriate IR version. We cannot use LLVM IR with SSE4.2 instructions on
@@ -375,9 +378,9 @@ void LlvmCodeGen::StripGlobalCtorsDtors(llvm::Module* module) {
if (destructors != NULL) destructors->eraseFromParent();
}
-Status LlvmCodeGen::CreateImpalaCodegen(
- ObjectPool* pool, const string& id, scoped_ptr<LlvmCodeGen>* codegen_ret) {
- RETURN_IF_ERROR(CreateFromMemory(pool, id, codegen_ret));
+Status LlvmCodeGen::CreateImpalaCodegen(ObjectPool* pool, MemTracker* parent_mem_tracker,
+ const string& id, scoped_ptr<LlvmCodeGen>* codegen_ret) {
+ RETURN_IF_ERROR(CreateFromMemory(pool, parent_mem_tracker, id, codegen_ret));
LlvmCodeGen* codegen = codegen_ret->get();
// Parse module for cross compiled functions and types
@@ -422,8 +425,9 @@ Status LlvmCodeGen::Init(unique_ptr<Module> module) {
EngineBuilder builder(std::move(module));
builder.setEngineKind(EngineKind::JIT);
builder.setOptLevel(opt_level);
- builder.setMCJITMemoryManager(
- unique_ptr<ImpalaMCJITMemoryManager>(new ImpalaMCJITMemoryManager()));
+ unique_ptr<ImpalaMCJITMemoryManager> memory_manager(new ImpalaMCJITMemoryManager);
+ memory_manager_ = memory_manager.get();
+ builder.setMCJITMemoryManager(move(memory_manager));
builder.setMCPU(cpu_name_);
builder.setMAttrs(cpu_attrs_);
builder.setErrorStr(&error_string_);
@@ -464,6 +468,10 @@ void LlvmCodeGen::SetupJITListeners() {
}
LlvmCodeGen::~LlvmCodeGen() {
+ if (memory_manager_ != NULL) mem_tracker_->Release(memory_manager_->bytes_tracked());
+ if (mem_tracker_->parent() != NULL) mem_tracker_->UnregisterFromParent();
+ mem_tracker_.reset();
+
// Execution engine executes callback on event listener, so tear down engine first.
execution_engine_.reset();
symbol_emitter_.reset();
@@ -753,8 +761,14 @@ bool LlvmCodeGen::VerifyFunction(Function* fn) {
return true;
}
-LlvmCodeGen::FnPrototype::FnPrototype(LlvmCodeGen* codegen, const string& name,
- Type* ret_type) : codegen_(codegen), name_(name), ret_type_(ret_type) {
+void LlvmCodeGen::SetNoInline(llvm::Function* function) const {
+ function->removeFnAttr(llvm::Attribute::AlwaysInline);
+ function->addFnAttr(llvm::Attribute::NoInline);
+}
+
+LlvmCodeGen::FnPrototype::FnPrototype(
+ LlvmCodeGen* codegen, const string& name, Type* ret_type)
+ : codegen_(codegen), name_(name), ret_type_(ret_type) {
DCHECK(!codegen_->is_compiled_) << "Not valid to add additional functions";
}
@@ -927,10 +941,15 @@ Status LlvmCodeGen::FinalizeModule() {
// Don't waste time optimizing module if there are no functions to JIT. This can happen
// if the codegen object is created but no functions are successfully codegen'd.
- if (fns_to_jit_compile_.empty()) return Status::OK();
+ if (fns_to_jit_compile_.empty()) {
+ DestroyModule();
+ return Status::OK();
+ }
RETURN_IF_ERROR(FinalizeLazyMaterialization());
- if (optimizations_enabled_ && !FLAGS_disable_optimization_passes) OptimizeModule();
+ if (optimizations_enabled_ && !FLAGS_disable_optimization_passes) {
+ RETURN_IF_ERROR(OptimizeModule());
+ }
if (FLAGS_opt_module_dir.size() != 0) {
string path = FLAGS_opt_module_dir + "/" + id_ + "_opt.ll";
@@ -949,17 +968,28 @@ Status LlvmCodeGen::FinalizeModule() {
execution_engine_->finalizeObject();
}
- // Get pointers to all codegen'd functions.
+ // Get pointers to all codegen'd functions
for (int i = 0; i < fns_to_jit_compile_.size(); ++i) {
Function* function = fns_to_jit_compile_[i].first;
void* jitted_function = execution_engine_->getPointerToFunction(function);
DCHECK(jitted_function != NULL) << "Failed to jit " << function->getName().data();
*fns_to_jit_compile_[i].second = jitted_function;
}
+
+ DestroyModule();
+
+ // Track the memory consumed by the compiled code.
+ int64_t bytes_allocated = memory_manager_->bytes_allocated();
+ if (!mem_tracker_->TryConsume(bytes_allocated)) {
+ const string& msg = Substitute(
+ "Failed to allocate '$0' bytes for compiled code module", bytes_allocated);
+ return mem_tracker_->MemLimitExceeded(NULL, msg, bytes_allocated);
+ }
+ memory_manager_->set_bytes_tracked(bytes_allocated);
return Status::OK();
}
-void LlvmCodeGen::OptimizeModule() {
+Status LlvmCodeGen::OptimizeModule() {
SCOPED_TIMER(optimization_timer_);
// This pass manager will construct optimizations passes that are "typical" for
@@ -1003,6 +1033,14 @@ void LlvmCodeGen::OptimizeModule() {
COUNTER_SET(num_functions_, counter.GetCount(InstructionCounter::TOTAL_FUNCTIONS));
COUNTER_SET(num_instructions_, counter.GetCount(InstructionCounter::TOTAL_INSTS));
+ int64_t estimated_memory = ESTIMATED_OPTIMIZER_BYTES_PER_INST
+ * counter.GetCount(InstructionCounter::TOTAL_INSTS);
+ if (!mem_tracker_->TryConsume(estimated_memory)) {
+ const string& msg = Substitute(
+ "Codegen failed to reserve '$0' bytes for optimization", estimated_memory);
+ return mem_tracker_->MemLimitExceeded(NULL, msg, estimated_memory);
+ }
+
// Create and run function pass manager
unique_ptr<legacy::FunctionPassManager> fn_pass_manager(
new legacy::FunctionPassManager(module_));
@@ -1027,6 +1065,22 @@ void LlvmCodeGen::OptimizeModule() {
VLOG(1) << counter.PrintCounters();
}
}
+
+ mem_tracker_->Release(estimated_memory);
+ return Status::OK();
+}
+
+void LlvmCodeGen::DestroyModule() {
+ // Clear all references to LLVM objects owned by the module.
+ loaded_functions_.clear();
+ codegend_functions_.clear();
+ registered_exprs_map_.clear();
+ registered_exprs_.clear();
+ llvm_intrinsics_.clear();
+ hash_fns_.clear();
+ fns_to_jit_compile_.clear();
+ execution_engine_->removeModule(module_);
+ module_ = NULL;
}
void LlvmCodeGen::AddFunctionToJit(Function* fn, void** fn_ptr) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/codegen/llvm-codegen.h
----------------------------------------------------------------------
diff --git a/be/src/codegen/llvm-codegen.h b/be/src/codegen/llvm-codegen.h
index 21f9f71..8853d7e 100644
--- a/be/src/codegen/llvm-codegen.h
+++ b/be/src/codegen/llvm-codegen.h
@@ -72,6 +72,7 @@ namespace llvm {
namespace impala {
class CodegenSymbolEmitter;
+class ImpalaMCJITMemoryManager;
class SubExprElimination;
class TupleDescriptor;
@@ -124,6 +125,14 @@ class LlvmBuilder : public llvm::IRBuilder<> {
/// instructions attached to the function object. Functions reachable by the function
/// are also materialized recursively.
//
+/// Memory used for codegen is tracked via the MemTracker hierarchy. Codegen can use
+/// significant memory for the IR module and for the optimization and compilation
+/// algorithms. LLVM provides no way to directly track this transient memory - instead
+/// the memory consumption is estimated based on the size of the IR module and released
+/// once compilation finishes. Once compilation finishes, the size of the compiled
+/// machine code is obtained from LLVM and and is tracked until the LlvmCodeGen object
+/// is torn down and the compiled code is freed.
+//
class LlvmCodeGen {
public:
/// This function must be called once per process before any llvm API calls are
@@ -138,13 +147,15 @@ class LlvmCodeGen {
/// Creates a codegen instance for Impala initialized with the cross-compiled Impala IR.
/// 'codegen' will contain the created object on success.
+ /// 'parent_mem_tracker' - if non-NULL, the CodeGen MemTracker is created under this.
/// 'id' is used for outputting the IR module for debugging.
- static Status CreateImpalaCodegen(
- ObjectPool*, const std::string& id, boost::scoped_ptr<LlvmCodeGen>* codegen);
+ static Status CreateImpalaCodegen(ObjectPool*, MemTracker* parent_mem_tracker,
+ const std::string& id, boost::scoped_ptr<LlvmCodeGen>* codegen);
/// Creates a LlvmCodeGen instance initialized with the module bitcode from 'file'.
/// 'codegen' will contain the created object on success.
- static Status CreateFromFile(ObjectPool*, const std::string& file, const std::string& id,
+ static Status CreateFromFile(ObjectPool*, MemTracker* parent_mem_tracker,
+ const std::string& file, const std::string& id,
boost::scoped_ptr<LlvmCodeGen>* codegen);
/// Removes all jit compiled dynamically linked functions from the process.
@@ -202,8 +213,8 @@ class LlvmCodeGen {
/// If 'print_ir' is true, the generated llvm::Function's IR will be printed when
/// GetIR() is called. Avoid doing so for IR function prototypes generated for
/// externally defined native function.
- llvm::Function* GeneratePrototype(LlvmBuilder* builder = NULL,
- llvm::Value** params = NULL, bool print_ir = true);
+ llvm::Function* GeneratePrototype(
+ LlvmBuilder* builder = NULL, llvm::Value** params = NULL, bool print_ir = true);
private:
friend class LlvmCodeGen;
@@ -266,7 +277,9 @@ class LlvmCodeGen {
/// Optimize and compile the module. This should be called after all functions to JIT
/// have been added to the module via AddFunctionToJit(). If optimizations_enabled_ is
- /// false, the module will not be optimized before compilation.
+ /// false, the module will not be optimized before compilation. After FinalizeModule()
+ /// is called, the LLVM module is destroyed and it is invalid to call any LlvmCodegen
+ /// functions.
Status FinalizeModule();
/// Replaces all instructions in 'caller' that call 'target_name' with a call
@@ -357,6 +370,10 @@ class LlvmCodeGen {
llvm::Function* GetFnvHashFunction(int num_bytes = -1);
llvm::Function* GetMurmurHashFunction(int num_bytes = -1);
+ /// Set the NoInline attribute on 'function' and remove the AlwaysInline attribute if
+ /// present.
+ void SetNoInline(llvm::Function* function) const;
+
/// Allocate stack storage for local variables. This is similar to traditional c, where
/// all the variables must be declared at the top of the function. This helper can be
/// called from anywhere and will add a stack allocation for 'var' at the beginning of
@@ -454,6 +471,15 @@ class LlvmCodeGen {
/// this LlvmCodeGen object. The module must be on the local filesystem.
Status LinkModule(const std::string& file);
+ /// If there are more than this number of expr trees (or functions that evaluate
+ /// expressions), avoid inlining avoid inlining for the exprs exceeding this threshold.
+ static const int CODEGEN_INLINE_EXPRS_THRESHOLD = 100;
+
+ /// If there are more than this number of expr trees (or functions that evaluate
+ /// expressions), avoid inlining the function that evaluates the expression batch
+ /// into the calling function.
+ static const int CODEGEN_INLINE_EXPR_BATCH_THRESHOLD = 25;
+
private:
friend class ExprCodegenTest;
friend class LlvmCodeGenTest;
@@ -471,11 +497,12 @@ class LlvmCodeGen {
/// Parses all the global variables in 'module' and adds any functions referenced by
/// them to the set 'ref_fns' if they are not defined in the Impalad native code.
/// These functions need to be materialized to avoid linking error.
- static void ParseGVForFunctions(llvm::Module* module,
- boost::unordered_set<string>* ref_fns);
+ static void ParseGVForFunctions(
+ llvm::Module* module, boost::unordered_set<string>* ref_fns);
/// Top level codegen object. 'module_id' is used for debugging when outputting the IR.
- LlvmCodeGen(ObjectPool* pool, const std::string& module_id);
+ LlvmCodeGen(
+ ObjectPool* pool, MemTracker* parent_mem_tracker, const std::string& module_id);
/// Initializes the jitter and execution engine with the given module.
Status Init(std::unique_ptr<llvm::Module> module);
@@ -484,8 +511,8 @@ class LlvmCodeGen {
/// 'codegen' will contain the created object on success. Note that the functions
/// are not materialized. Getting a reference to the function via GetFunction()
/// will materialize the function and its callees recursively.
- static Status CreateFromMemory(ObjectPool* pool, const std::string& id,
- boost::scoped_ptr<LlvmCodeGen>* codegen);
+ static Status CreateFromMemory(ObjectPool* pool, MemTracker* parent_mem_tracker,
+ const std::string& id, boost::scoped_ptr<LlvmCodeGen>* codegen);
/// Loads an LLVM module from 'file' which is the local path to the LLVM bitcode file.
/// The functions in the module are not materialized. Getting a reference to the
@@ -524,7 +551,7 @@ class LlvmCodeGen {
void ResetVerification() { is_corrupt_ = false; }
/// Optimizes the module. This includes pruning the module of any unused functions.
- void OptimizeModule();
+ Status OptimizeModule();
/// Clears generated hash fns. This is only used for testing.
void ClearHashFns();
@@ -570,6 +597,10 @@ class LlvmCodeGen {
/// there is error in materializing the module.
Status FinalizeLazyMaterialization();
+ /// Destroy the IR module, freeing memory used by the IR. Any machine code that was
+ /// generated is retained by the execution engine.
+ void DestroyModule();
+
/// Whether InitializeLlvm() has been called.
static bool llvm_initialized_;
@@ -589,6 +620,10 @@ class LlvmCodeGen {
/// Codegen counters
RuntimeProfile profile_;
+ /// MemTracker used for tracking memory consumed by codegen. Connected to a parent
+ /// MemTracker if one was provided during initialization.
+ boost::scoped_ptr<MemTracker> mem_tracker_;
+
/// Time spent reading the .ir file from the file system.
RuntimeProfile::Counter* load_module_timer_;
@@ -639,6 +674,9 @@ class LlvmCodeGen {
/// Execution/Jitting engine.
std::unique_ptr<llvm::ExecutionEngine> execution_engine_;
+ /// The memory manager used by 'execution_engine_'. Owned by 'execution_engine_'.
+ ImpalaMCJITMemoryManager* memory_manager_;
+
/// Functions parsed from pre-compiled module. Indexed by ImpalaIR::Function enum
std::vector<llvm::Function*> loaded_functions_;
@@ -686,8 +724,28 @@ class LlvmCodeGen {
/// 'symbol_emitter_' are called by 'execution_engine_' when code is emitted or freed.
/// The lifetime of the symbol emitter must be longer than 'execution_engine_'.
boost::scoped_ptr<CodegenSymbolEmitter> symbol_emitter_;
-};
+ /// Very rough estimate of memory in bytes that the IR and the intermediate data
+ /// structures used by the optimizer may consume per LLVM IR instruction to be
+ /// optimized (after dead code is removed). The number is chosen to avoid pathological
+ /// behaviour at either extreme: failing queries unnecessarily because the memory
+ /// estimate is too high versus having large amounts of untracked memory because the
+ /// estimate is too low.
+ ///
+ /// This was chosen by looking at the behaviour of TPC-H queries. Using the heap growth
+ /// profile from gperftools reveal that LLVM allocated ~9mb of memory for fragments with
+ /// ~17k total instructions in TPC-H Q2. Inspection of other TPC-H queries revealed
+ /// that a typical fragment from a TPC-H query is < 5,000 instructions, which translates
+ /// to 2.5MB, which is almost always lower than the runtime memory requirement of the
+ /// fragment - so we are unlikely to fail queries unnecessarily.
+ ///
+ /// This assumes optimizer memory usage scales linearly with instruction count. This is
+ /// true only if the size of functions is bounded, because some optimization passes
+ /// (e.g. global value numbering) use time and memory that is super-linear in relation
+ /// to the # of instructions in a function. So codegen should avoid generating
+ /// arbitrarily large function.
+ static constexpr int64_t ESTIMATED_OPTIMIZER_BYTES_PER_INST = 512;
+};
}
#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/codegen/mcjit-mem-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/codegen/mcjit-mem-mgr.h b/be/src/codegen/mcjit-mem-mgr.h
index 313d175..6982cef 100644
--- a/be/src/codegen/mcjit-mem-mgr.h
+++ b/be/src/codegen/mcjit-mem-mgr.h
@@ -25,19 +25,53 @@ extern void *__dso_handle __attribute__ ((__visibility__ ("hidden")));
namespace impala {
-/// Custom memory manager to resolve references to __dso_handle in cross-compiled IR.
+/// Custom memory manager. It is needed for a couple of purposes.
+///
+/// We use it as a way to resolve references to __dso_handle in cross-compiled IR.
/// This uses the same approach as the legacy llvm JIT to handle __dso_handle. MCJIT
/// doesn't handle those for us: see LLVM issue 18062.
/// TODO: get rid of this by purging the cross-compiled IR of references to __dso_handle,
/// which come from global variables with destructors.
+///
+/// We also use it to track how much memory is allocated for compiled code.
class ImpalaMCJITMemoryManager : public llvm::SectionMemoryManager {
public:
+ ImpalaMCJITMemoryManager() : bytes_allocated_(0), bytes_tracked_(0){};
+
virtual uint64_t getSymbolAddress(const std::string& name) override {
if (name == "__dso_handle") return reinterpret_cast<uint64_t>(&__dso_handle);
return SectionMemoryManager::getSymbolAddress(name);
}
-};
+ virtual uint8_t* allocateCodeSection(uintptr_t size, unsigned alignment,
+ unsigned section_id, llvm::StringRef section_name) override {
+ bytes_allocated_ += size;
+ return llvm::SectionMemoryManager::allocateCodeSection(
+ size, alignment, section_id, section_name);
+ }
+
+ virtual uint8_t* allocateDataSection(uintptr_t size, unsigned alignment,
+ unsigned section_id, llvm::StringRef section_name, bool is_read_only) override {
+ bytes_allocated_ += size;
+ return llvm::SectionMemoryManager::allocateDataSection(
+ size, alignment, section_id, section_name, is_read_only);
+ }
+
+ int64_t bytes_allocated() const { return bytes_allocated_; }
+ int64_t bytes_tracked() const { return bytes_tracked_; }
+ void set_bytes_tracked(int64_t bytes_tracked) {
+ DCHECK_LE(bytes_tracked, bytes_allocated_);
+ bytes_tracked_ = bytes_tracked;
+ }
+
+ private:
+ /// Total bytes allocated for the compiled code.
+ int64_t bytes_allocated_;
+
+ /// Total bytes already tracked by MemTrackers. <= 'bytes_allocated_'.
+ /// Needed to release the correct amount from the MemTracker when done.
+ int64_t bytes_tracked_;
+};
}
#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/aggregation-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/aggregation-node.cc b/be/src/exec/aggregation-node.cc
index f3b054e..822fc5d 100644
--- a/be/src/exec/aggregation-node.cc
+++ b/be/src/exec/aggregation-node.cc
@@ -164,6 +164,9 @@ Status AggregationNode::Prepare(RuntimeState* state) {
void AggregationNode::Codegen(RuntimeState* state) {
DCHECK(state->ShouldCodegen());
+ ExecNode::Codegen(state);
+ if (IsNodeCodegenDisabled()) return;
+
bool codegen_enabled = false;
LlvmCodeGen* codegen = state->codegen();
DCHECK(codegen != NULL);
@@ -178,7 +181,6 @@ void AggregationNode::Codegen(RuntimeState* state) {
}
}
runtime_profile()->AddCodegenMsg(codegen_enabled);
- ExecNode::Codegen(state);
}
Status AggregationNode::Open(RuntimeState* state) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/exchange-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/exchange-node.cc b/be/src/exec/exchange-node.cc
index 2b2b033..0bbd42b 100644
--- a/be/src/exec/exchange-node.cc
+++ b/be/src/exec/exchange-node.cc
@@ -92,11 +92,13 @@ Status ExchangeNode::Prepare(RuntimeState* state) {
void ExchangeNode::Codegen(RuntimeState* state) {
DCHECK(state->ShouldCodegen());
+ ExecNode::Codegen(state);
+ if (IsNodeCodegenDisabled()) return;
+
if (is_merging_) {
Status codegen_status = less_than_->Codegen(state);
runtime_profile()->AddCodegenMsg(codegen_status.ok(), codegen_status);
}
- ExecNode::Codegen(state);
}
Status ExchangeNode::Open(RuntimeState* state) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/exec-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/exec-node.cc b/be/src/exec/exec-node.cc
index 95306ec..518a0dd 100644
--- a/be/src/exec/exec-node.cc
+++ b/be/src/exec/exec-node.cc
@@ -129,6 +129,7 @@ ExecNode::ExecNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl
rows_returned_counter_(NULL),
rows_returned_rate_(NULL),
containing_subplan_(NULL),
+ disable_codegen_(tnode.disable_codegen),
is_closed_(false) {
InitRuntimeProfile(PrintPlanNodeType(tnode.node_type));
}
@@ -465,6 +466,10 @@ void ExecNode::AddCodegenDisabledMessage(RuntimeState* state) {
}
}
+bool ExecNode::IsNodeCodegenDisabled() const {
+ return disable_codegen_;
+}
+
// Codegen for EvalConjuncts. The generated signature is
// For a node with two conjunct predicates
// define i1 @EvalConjuncts(%"class.impala::ExprContext"** %ctxs, i32 %num_ctxs,
@@ -507,6 +512,10 @@ Status ExecNode::CodegenEvalConjuncts(LlvmCodeGen* codegen,
for (int i = 0; i < conjunct_ctxs.size(); ++i) {
RETURN_IF_ERROR(
conjunct_ctxs[i]->root()->GetCodegendComputeFn(codegen, &conjunct_fns[i]));
+ if (i >= LlvmCodeGen::CODEGEN_INLINE_EXPRS_THRESHOLD) {
+ // Avoid bloating EvalConjuncts by inlining everything into it.
+ codegen->SetNoInline(conjunct_fns[i]);
+ }
}
// Construct function signature to match
@@ -566,12 +575,16 @@ Status ExecNode::CodegenEvalConjuncts(LlvmCodeGen* codegen,
builder.CreateRet(codegen->true_value());
}
+ // Avoid inlining EvalConjuncts into caller if it is large.
+ if (conjunct_ctxs.size() > LlvmCodeGen::CODEGEN_INLINE_EXPR_BATCH_THRESHOLD) {
+ codegen->SetNoInline(*fn);
+ }
+
*fn = codegen->FinalizeFunction(*fn);
if (*fn == NULL) {
return Status("ExecNode::CodegenEvalConjuncts(): codegen'd EvalConjuncts() function "
- "failed verification, see log");
+ "failed verification, see log");
}
return Status::OK();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/exec-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/exec-node.h b/be/src/exec/exec-node.h
index 1dd2d32..6a7737b 100644
--- a/be/src/exec/exec-node.h
+++ b/be/src/exec/exec-node.h
@@ -184,6 +184,10 @@ class ExecNode {
MemTracker* mem_tracker() { return mem_tracker_.get(); }
MemTracker* expr_mem_tracker() { return expr_mem_tracker_.get(); }
+ /// Return true if codegen was disabled by the planner for this ExecNode. Does not
+ /// check to see if codegen was enabled for the enclosing fragment.
+ bool IsNodeCodegenDisabled() const;
+
/// Add codegen disabled message if codegen is disabled for this ExecNode.
void AddCodegenDisabledMessage(RuntimeState* state);
@@ -276,6 +280,9 @@ class ExecNode {
/// Valid to call in or after Prepare().
bool IsInSubplan() const { return containing_subplan_ != NULL; }
+ /// If true, codegen should be disabled for this exec node.
+ const bool disable_codegen_;
+
/// Create a single exec node derived from thrift node; place exec node in 'pool'.
static Status CreateNode(ObjectPool* pool, const TPlanNode& tnode,
const DescriptorTbl& descs, ExecNode** node, RuntimeState* state);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/hash-join-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hash-join-node.cc b/be/src/exec/hash-join-node.cc
index a18f356..77807f6 100644
--- a/be/src/exec/hash-join-node.cc
+++ b/be/src/exec/hash-join-node.cc
@@ -134,14 +134,13 @@ Status HashJoinNode::Prepare(RuntimeState* state) {
AddExprCtxsToFree(other_join_conjunct_ctxs_);
// TODO: default buckets
- const bool stores_nulls = join_op_ == TJoinOp::RIGHT_OUTER_JOIN ||
- join_op_ == TJoinOp::FULL_OUTER_JOIN ||
- std::accumulate(is_not_distinct_from_.begin(), is_not_distinct_from_.end(), false,
- std::logical_or<bool>());
+ const bool stores_nulls = join_op_ == TJoinOp::RIGHT_OUTER_JOIN
+ || join_op_ == TJoinOp::FULL_OUTER_JOIN
+ || std::accumulate(is_not_distinct_from_.begin(), is_not_distinct_from_.end(),
+ false, std::logical_or<bool>());
hash_tbl_.reset(new OldHashTable(state, build_expr_ctxs_, probe_expr_ctxs_,
- filter_expr_ctxs_,
- child(1)->row_desc().tuple_descriptors().size(), stores_nulls,
- is_not_distinct_from_, state->fragment_hash_seed(), mem_tracker(), filters_));
+ filter_expr_ctxs_, child(1)->row_desc().tuple_descriptors().size(), stores_nulls,
+ is_not_distinct_from_, state->fragment_hash_seed(), mem_tracker(), filters_));
build_pool_.reset(new MemPool(mem_tracker()));
AddCodegenDisabledMessage(state);
return Status::OK();
@@ -149,6 +148,9 @@ Status HashJoinNode::Prepare(RuntimeState* state) {
void HashJoinNode::Codegen(RuntimeState* state) {
DCHECK(state->ShouldCodegen());
+ ExecNode::Codegen(state);
+ if (IsNodeCodegenDisabled()) return;
+
LlvmCodeGen* codegen = state->codegen();
DCHECK(codegen != NULL);
bool build_codegen_enabled = false;
@@ -178,7 +180,6 @@ void HashJoinNode::Codegen(RuntimeState* state) {
}
runtime_profile()->AddCodegenMsg(build_codegen_enabled, "", "Build Side");
runtime_profile()->AddCodegenMsg(probe_codegen_enabled, "", "Probe Side");
- ExecNode::Codegen(state);
}
Status HashJoinNode::Reset(RuntimeState* state) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/hash-table.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hash-table.cc b/be/src/exec/hash-table.cc
index 8d45ec7..f4ecb67 100644
--- a/be/src/exec/hash-table.cc
+++ b/be/src/exec/hash-table.cc
@@ -755,13 +755,18 @@ Status HashTableCtx::CodegenEvalRow(LlvmCodeGen* codegen, bool build, Function**
if (!status.ok()) {
(*fn)->eraseFromParent(); // deletes function
*fn = NULL;
- return Status(Substitute("Problem with HashTableCtx::CodegenEvalRow(): $0",
- status.GetDetail()));
+ return Status(Substitute(
+ "Problem with HashTableCtx::CodegenEvalRow(): $0", status.GetDetail()));
}
- Value* get_expr_ctx_args[] = { this_ptr, codegen->GetIntConstant(TYPE_INT, i) };
+ // Avoid bloating function by inlining too many exprs into it.
+ if (i >= LlvmCodeGen::CODEGEN_INLINE_EXPRS_THRESHOLD) {
+ codegen->SetNoInline(expr_fn);
+ }
+
+ Value* get_expr_ctx_args[] = {this_ptr, codegen->GetIntConstant(TYPE_INT, i)};
Value* ctx_arg = builder.CreateCall(get_expr_ctx_fn, get_expr_ctx_args, "expr_ctx");
- Value* expr_fn_args[] = { ctx_arg, row };
+ Value* expr_fn_args[] = {ctx_arg, row};
CodegenAnyVal result = CodegenAnyVal::CreateCallWrapped(
codegen, &builder, ctxs[i]->root()->type(), expr_fn, expr_fn_args, "result");
Value* is_null = result.GetIsNull();
@@ -800,10 +805,15 @@ Status HashTableCtx::CodegenEvalRow(LlvmCodeGen* codegen, bool build, Function**
}
builder.CreateRet(has_null);
+ // Avoid inlining a large EvalRow() function into caller.
+ if (ctxs.size() > LlvmCodeGen::CODEGEN_INLINE_EXPR_BATCH_THRESHOLD) {
+ codegen->SetNoInline(*fn);
+ }
+
*fn = codegen->FinalizeFunction(*fn);
if (*fn == NULL) {
return Status("Codegen'd HashTableCtx::EvalRow() function failed verification, "
- "see log");
+ "see log");
}
return Status::OK();
}
@@ -972,6 +982,11 @@ Status HashTableCtx::CodegenHashRow(LlvmCodeGen* codegen, bool use_murmur, Funct
}
builder.CreateRet(hash_result);
+
+ // Avoid inlining into caller if there are many exprs.
+ if (build_expr_ctxs_.size() > LlvmCodeGen::CODEGEN_INLINE_EXPR_BATCH_THRESHOLD) {
+ codegen->SetNoInline(*fn);
+ }
*fn = codegen->FinalizeFunction(*fn);
if (*fn == NULL) {
return Status(
@@ -1095,12 +1110,16 @@ Status HashTableCtx::CodegenEquals(LlvmCodeGen* codegen, bool force_null_equalit
if (!status.ok()) {
(*fn)->eraseFromParent(); // deletes function
*fn = NULL;
- return Status(Substitute("Problem with HashTableCtx::CodegenEquals: $0",
- status.GetDetail()));
+ return Status(
+ Substitute("Problem with HashTableCtx::CodegenEquals: $0", status.GetDetail()));
+ }
+ if (build_expr_ctxs_.size() > LlvmCodeGen::CODEGEN_INLINE_EXPRS_THRESHOLD) {
+ // Avoid bloating function by inlining too many exprs into it.
+ codegen->SetNoInline(expr_fn);
}
// Load ExprContext* from 'build_expr_ctxs_'.
- Value* get_expr_ctx_args[] = { this_ptr, codegen->GetIntConstant(TYPE_INT, i) };
+ Value* get_expr_ctx_args[] = {this_ptr, codegen->GetIntConstant(TYPE_INT, i)};
Value* ctx_arg = builder.CreateCall(get_expr_ctx_fn, get_expr_ctx_args, "expr_ctx");
// Evaluate the expression.
@@ -1156,10 +1175,14 @@ Status HashTableCtx::CodegenEquals(LlvmCodeGen* codegen, bool force_null_equalit
builder.SetInsertPoint(false_block);
builder.CreateRet(codegen->false_value());
+ // Avoid inlining into caller if it is large.
+ if (build_expr_ctxs_.size() > LlvmCodeGen::CODEGEN_INLINE_EXPR_BATCH_THRESHOLD) {
+ codegen->SetNoInline(*fn);
+ }
*fn = codegen->FinalizeFunction(*fn);
if (*fn == NULL) {
return Status("Codegen'd HashTableCtx::Equals() function failed verification, "
- "see log");
+ "see log");
}
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/hdfs-scan-node-base.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-base.cc b/be/src/exec/hdfs-scan-node-base.cc
index e616faf..5b97e76 100644
--- a/be/src/exec/hdfs-scan-node-base.cc
+++ b/be/src/exec/hdfs-scan-node-base.cc
@@ -297,6 +297,10 @@ Status HdfsScanNodeBase::Prepare(RuntimeState* state) {
}
void HdfsScanNodeBase::Codegen(RuntimeState* state) {
+ DCHECK(state->ShouldCodegen());
+ ExecNode::Codegen(state);
+ if (IsNodeCodegenDisabled()) return;
+
// Create codegen'd functions
for (int format = THdfsFileFormat::TEXT; format <= THdfsFileFormat::PARQUET; ++format) {
vector<HdfsFileDesc*>& file_descs =
@@ -343,7 +347,6 @@ void HdfsScanNodeBase::Codegen(RuntimeState* state) {
}
runtime_profile()->AddCodegenMsg(status.ok(), status, format_name);
}
- ExecNode::Codegen(state);
}
Status HdfsScanNodeBase::Open(RuntimeState* state) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/hdfs-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scanner.cc b/be/src/exec/hdfs-scanner.cc
index 390085f..c6a5f37 100644
--- a/be/src/exec/hdfs-scanner.cc
+++ b/be/src/exec/hdfs-scanner.cc
@@ -332,6 +332,7 @@ Status HdfsScanner::CodegenWriteCompleteTuple(HdfsScanNodeBase* node,
node->hdfs_table()->null_column_value().data(),
node->hdfs_table()->null_column_value().size(), true, state->strict_mode());
if (fn == NULL) return Status("CodegenWriteSlot failed.");
+ if (i >= LlvmCodeGen::CODEGEN_INLINE_EXPRS_THRESHOLD) codegen->SetNoInline(fn);
slot_fns.push_back(fn);
}
@@ -487,6 +488,10 @@ Status HdfsScanner::CodegenWriteCompleteTuple(HdfsScanNodeBase* node,
fn->eraseFromParent();
return status;
}
+ if (node->materialized_slots().size() + conjunct_idx
+ >= LlvmCodeGen::CODEGEN_INLINE_EXPRS_THRESHOLD) {
+ codegen->SetNoInline(conjunct_fn);
+ }
Function* get_ctx_fn =
codegen->GetFunction(IRFunction::HDFS_SCANNER_GET_CONJUNCT_CTX, false);
@@ -505,6 +510,10 @@ Status HdfsScanner::CodegenWriteCompleteTuple(HdfsScanNodeBase* node,
builder.SetInsertPoint(eval_fail_block);
builder.CreateRet(codegen->false_value());
+ if (node->materialized_slots().size() + conjunct_ctxs.size()
+ > LlvmCodeGen::CODEGEN_INLINE_EXPR_BATCH_THRESHOLD) {
+ codegen->SetNoInline(fn);
+ }
*write_complete_tuple_fn = codegen->FinalizeFunction(fn);
if (*write_complete_tuple_fn == NULL) {
return Status("Failed to finalize write_complete_tuple_fn.");
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/partitioned-aggregation-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/partitioned-aggregation-node.cc b/be/src/exec/partitioned-aggregation-node.cc
index 01622b9..6cc36be 100644
--- a/be/src/exec/partitioned-aggregation-node.cc
+++ b/be/src/exec/partitioned-aggregation-node.cc
@@ -281,14 +281,16 @@ Status PartitionedAggregationNode::Prepare(RuntimeState* state) {
void PartitionedAggregationNode::Codegen(RuntimeState* state) {
DCHECK(state->ShouldCodegen());
+ ExecNode::Codegen(state);
+ if (IsNodeCodegenDisabled()) return;
+
LlvmCodeGen* codegen = state->codegen();
DCHECK(codegen != NULL);
TPrefetchMode::type prefetch_mode = state_->query_options().prefetch_mode;
- Status codegen_status =
- is_streaming_preagg_ ? CodegenProcessBatchStreaming(codegen, prefetch_mode) :
- CodegenProcessBatch(codegen, prefetch_mode);
+ Status codegen_status = is_streaming_preagg_ ?
+ CodegenProcessBatchStreaming(codegen, prefetch_mode) :
+ CodegenProcessBatch(codegen, prefetch_mode);
runtime_profile()->AddCodegenMsg(codegen_status.ok(), codegen_status);
- ExecNode::Codegen(state);
}
Status PartitionedAggregationNode::Open(RuntimeState* state) {
@@ -1553,7 +1555,8 @@ Status PartitionedAggregationNode::QueryMaintenance(RuntimeState* state) {
// }
//
Status PartitionedAggregationNode::CodegenUpdateSlot(LlvmCodeGen* codegen,
- AggFnEvaluator* evaluator, SlotDescriptor* slot_desc, Function** fn) {
+ AggFnEvaluator* evaluator, int evaluator_idx, SlotDescriptor* slot_desc,
+ Function** fn) {
PointerType* fn_ctx_type =
codegen->GetPtrType(FunctionContextImpl::LLVM_FUNCTIONCONTEXT_NAME);
PointerType* expr_ctxs_type =
@@ -1697,10 +1700,18 @@ Status PartitionedAggregationNode::CodegenUpdateSlot(LlvmCodeGen* codegen,
builder.SetInsertPoint(ret_block);
builder.CreateRetVoid();
+ // Avoid producing huge UpdateTuple() function after inlining - LLVM's optimiser
+ // memory/CPU usage scales super-linearly with function size.
+ // E.g. compute stats on all columns of a 1000-column table previously took 4 minutes to
+ // codegen because all the UpdateSlot() functions were inlined.
+ if (evaluator_idx >= LlvmCodeGen::CODEGEN_INLINE_EXPRS_THRESHOLD) {
+ codegen->SetNoInline(*fn);
+ }
+
*fn = codegen->FinalizeFunction(*fn);
if (*fn == NULL) {
return Status("PartitionedAggregationNode::CodegenUpdateSlot(): codegen'd "
- "UpdateSlot() function failed verification, see log");
+ "UpdateSlot() function failed verification, see log");
}
return Status::OK();
}
@@ -1874,7 +1885,8 @@ Status PartitionedAggregationNode::CodegenUpdateTuple(
builder.CreateStore(count_inc, slot_ptr);
} else {
Function* update_slot_fn;
- RETURN_IF_ERROR(CodegenUpdateSlot(codegen, evaluator, slot_desc, &update_slot_fn));
+ RETURN_IF_ERROR(
+ CodegenUpdateSlot(codegen, evaluator, i, slot_desc, &update_slot_fn));
Value* agg_fn_ctx_ptr = builder.CreateConstGEP1_32(agg_fn_ctxs_arg, i);
Value* agg_fn_ctx = builder.CreateLoad(agg_fn_ctx_ptr, "agg_fn_ctx");
// Call GetExprCtx() to get the expression context.
@@ -1887,6 +1899,12 @@ Status PartitionedAggregationNode::CodegenUpdateTuple(
}
builder.CreateRetVoid();
+ // Avoid inlining big UpdateTuple function into outer loop - we're unlikely to get
+ // any benefit from it since the function call overhead will be amortized.
+ if (aggregate_evaluators_.size() > LlvmCodeGen::CODEGEN_INLINE_EXPR_BATCH_THRESHOLD) {
+ codegen->SetNoInline(*fn);
+ }
+
// CodegenProcessBatch() does the final optimizations.
*fn = codegen->FinalizeFunction(*fn);
if (*fn == NULL) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/partitioned-aggregation-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/partitioned-aggregation-node.h b/be/src/exec/partitioned-aggregation-node.h
index 4287cea..54840c7 100644
--- a/be/src/exec/partitioned-aggregation-node.h
+++ b/be/src/exec/partitioned-aggregation-node.h
@@ -643,7 +643,7 @@ class PartitionedAggregationNode : public ExecNode {
/// Codegen UpdateSlot(). Returns non-OK status if codegen is unsuccessful.
/// Assumes is_merge = false;
Status CodegenUpdateSlot(LlvmCodeGen* codegen, AggFnEvaluator* evaluator,
- SlotDescriptor* slot_desc, llvm::Function** fn);
+ int evaluator_idx, SlotDescriptor* slot_desc, llvm::Function** fn);
/// Codegen a call to a function implementing the UDA interface with input values
/// from 'input_vals'. 'dst_val' should contain the previous value of the aggregate
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/partitioned-hash-join-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/partitioned-hash-join-node.cc b/be/src/exec/partitioned-hash-join-node.cc
index 61b7eb7..6073486 100644
--- a/be/src/exec/partitioned-hash-join-node.cc
+++ b/be/src/exec/partitioned-hash-join-node.cc
@@ -141,6 +141,10 @@ Status PartitionedHashJoinNode::Prepare(RuntimeState* state) {
void PartitionedHashJoinNode::Codegen(RuntimeState* state) {
DCHECK(state->ShouldCodegen());
+ // Codegen the children node;
+ ExecNode::Codegen(state);
+ if (IsNodeCodegenDisabled()) return;
+
LlvmCodeGen* codegen = state->codegen();
DCHECK(codegen != NULL);
@@ -150,11 +154,8 @@ void PartitionedHashJoinNode::Codegen(RuntimeState* state) {
// Codegen the probe side.
TPrefetchMode::type prefetch_mode = state->query_options().prefetch_mode;
Status probe_codegen_status = CodegenProcessProbeBatch(codegen, prefetch_mode);
- runtime_profile()->AddCodegenMsg(probe_codegen_status.ok(), probe_codegen_status,
- "Probe Side");
-
- // Codegen the children node;
- ExecNode::Codegen(state);
+ runtime_profile()->AddCodegenMsg(
+ probe_codegen_status.ok(), probe_codegen_status, "Probe Side");
}
Status PartitionedHashJoinNode::Open(RuntimeState* state) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/sort-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/sort-node.cc b/be/src/exec/sort-node.cc
index 7518223..1a8de83 100644
--- a/be/src/exec/sort-node.cc
+++ b/be/src/exec/sort-node.cc
@@ -51,8 +51,9 @@ Status SortNode::Prepare(RuntimeState* state) {
state, child(0)->row_desc(), row_descriptor_, expr_mem_tracker()));
AddExprCtxsToFree(sort_exec_exprs_);
less_than_.reset(new TupleRowComparator(sort_exec_exprs_, is_asc_order_, nulls_first_));
- sorter_.reset(new Sorter(*less_than_.get(), sort_exec_exprs_.sort_tuple_slot_expr_ctxs(),
- &row_descriptor_, mem_tracker(), runtime_profile(), state));
+ sorter_.reset(
+ new Sorter(*less_than_.get(), sort_exec_exprs_.sort_tuple_slot_expr_ctxs(),
+ &row_descriptor_, mem_tracker(), runtime_profile(), state));
RETURN_IF_ERROR(sorter_->Init());
AddCodegenDisabledMessage(state);
return Status::OK();
@@ -60,9 +61,11 @@ Status SortNode::Prepare(RuntimeState* state) {
void SortNode::Codegen(RuntimeState* state) {
DCHECK(state->ShouldCodegen());
+ ExecNode::Codegen(state);
+ if (IsNodeCodegenDisabled()) return;
+
Status codegen_status = less_than_->Codegen(state);
runtime_profile()->AddCodegenMsg(codegen_status.ok(), codegen_status);
- ExecNode::Codegen(state);
}
Status SortNode::Open(RuntimeState* state) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exec/topn-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/topn-node.cc b/be/src/exec/topn-node.cc
index bd4caec..a7f11bd 100644
--- a/be/src/exec/topn-node.cc
+++ b/be/src/exec/topn-node.cc
@@ -84,6 +84,9 @@ Status TopNNode::Prepare(RuntimeState* state) {
void TopNNode::Codegen(RuntimeState* state) {
DCHECK(state->ShouldCodegen());
+ ExecNode::Codegen(state);
+ if (IsNodeCodegenDisabled()) return;
+
LlvmCodeGen* codegen = state->codegen();
DCHECK(codegen != NULL);
@@ -126,7 +129,6 @@ void TopNNode::Codegen(RuntimeState* state) {
}
}
runtime_profile()->AddCodegenMsg(codegen_status.ok(), codegen_status);
- ExecNode::Codegen(state);
}
Status TopNNode::Open(RuntimeState* state) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exprs/aggregate-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/aggregate-functions-ir.cc b/be/src/exprs/aggregate-functions-ir.cc
index 7ed6386..f6a91f4 100644
--- a/be/src/exprs/aggregate-functions-ir.cc
+++ b/be/src/exprs/aggregate-functions-ir.cc
@@ -1190,8 +1190,26 @@ void AggregateFunctions::HllUpdate(FunctionContext* ctx, const T& src, StringVal
}
}
-void AggregateFunctions::HllMerge(FunctionContext* ctx, const StringVal& src,
- StringVal* dst) {
+// Specialize for DecimalVal to allow substituting decimal size.
+template <>
+void AggregateFunctions::HllUpdate(
+ FunctionContext* ctx, const DecimalVal& src, StringVal* dst) {
+ if (src.is_null) return;
+ DCHECK(!dst->is_null);
+ DCHECK_EQ(dst->len, HLL_LEN);
+ uint64_t hash_value = AnyValUtil::HashDecimal64(
+ src, Expr::GetConstantInt(*ctx, Expr::ARG_TYPE_SIZE, 0), HashUtil::FNV64_SEED);
+ if (hash_value != 0) {
+ // Use the lower bits to index into the number of streams and then
+ // find the first 1 bit after the index bits.
+ int idx = hash_value & (HLL_LEN - 1);
+ uint8_t first_one_bit = __builtin_ctzl(hash_value >> HLL_PRECISION) + 1;
+ dst->ptr[idx] = ::max(dst->ptr[idx], first_one_bit);
+ }
+}
+
+void AggregateFunctions::HllMerge(
+ FunctionContext* ctx, const StringVal& src, StringVal* dst) {
DCHECK(!dst->is_null);
DCHECK(!src.is_null);
DCHECK_EQ(dst->len, HLL_LEN);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exprs/anyval-util.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/anyval-util.h b/be/src/exprs/anyval-util.h
index 8c75247..4703266 100644
--- a/be/src/exprs/anyval-util.h
+++ b/be/src/exprs/anyval-util.h
@@ -133,12 +133,19 @@ class AnyValUtil {
return HashUtil::MurmurHash2_64(&tv, 12, seed);
}
- static uint64_t Hash64(const DecimalVal& v, const FunctionContext::TypeDesc& t,
- int64_t seed) {
- switch (ColumnType::GetDecimalByteSize(t.precision)) {
- case 4: return HashUtil::MurmurHash2_64(&v.val4, 4, seed);
- case 8: return HashUtil::MurmurHash2_64(&v.val8, 8, seed);
- case 16: return HashUtil::MurmurHash2_64(&v.val16, 16, seed);
+ static uint64_t Hash64(
+ const DecimalVal& v, const FunctionContext::TypeDesc& t, int64_t seed) {
+ return HashDecimal64(v, ColumnType::GetDecimalByteSize(t.precision), seed);
+ }
+
+ static uint64_t HashDecimal64(const DecimalVal& v, int byte_size, int64_t seed) {
+ switch (byte_size) {
+ case 4:
+ return HashUtil::MurmurHash2_64(&v.val4, 4, seed);
+ case 8:
+ return HashUtil::MurmurHash2_64(&v.val8, 8, seed);
+ case 16:
+ return HashUtil::MurmurHash2_64(&v.val16, 16, seed);
default:
DCHECK(false);
return 0;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exprs/expr-codegen-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-codegen-test.cc b/be/src/exprs/expr-codegen-test.cc
index 6ccbbf4..cbb5617 100644
--- a/be/src/exprs/expr-codegen-test.cc
+++ b/be/src/exprs/expr-codegen-test.cc
@@ -251,7 +251,8 @@ TEST_F(ExprCodegenTest, TestInlineConstants) {
stringstream test_udf_file;
test_udf_file << getenv("IMPALA_HOME") << "/be/build/latest/exprs/expr-codegen-test.ll";
scoped_ptr<LlvmCodeGen> codegen;
- ASSERT_OK(LlvmCodeGen::CreateFromFile(&pool, test_udf_file.str(), "test", &codegen));
+ ASSERT_OK(
+ LlvmCodeGen::CreateFromFile(&pool, NULL, test_udf_file.str(), "test", &codegen));
Function* fn = codegen->GetFunction(TEST_GET_CONSTANT_SYMBOL, false);
ASSERT_TRUE(fn != NULL);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/exprs/expr.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr.h b/be/src/exprs/expr.h
index 552fee9..942eec0 100644
--- a/be/src/exprs/expr.h
+++ b/be/src/exprs/expr.h
@@ -361,6 +361,7 @@ class Expr {
/// recognize if this node is a slotref in order to speed up GetValue()
const bool is_slotref_;
+
/// analysis is done, types are fixed at this point
const ColumnType type_;
std::vector<Expr*> children_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/runtime/lib-cache.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/lib-cache.cc b/be/src/runtime/lib-cache.cc
index fe8d180..00c1159 100644
--- a/be/src/runtime/lib-cache.cc
+++ b/be/src/runtime/lib-cache.cc
@@ -403,8 +403,8 @@ Status LibCache::GetCacheEntryInternal(const string& hdfs_lib_file, LibType type
ObjectPool pool;
scoped_ptr<LlvmCodeGen> codegen;
string module_id = filesystem::path((*entry)->local_path).stem().string();
- RETURN_IF_ERROR(
- LlvmCodeGen::CreateFromFile(&pool, (*entry)->local_path, module_id, &codegen));
+ RETURN_IF_ERROR(LlvmCodeGen::CreateFromFile(
+ &pool, NULL, (*entry)->local_path, module_id, &codegen));
codegen->GetSymbols(&(*entry)->symbols);
} else {
DCHECK_EQ(type, TYPE_JAR);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/be/src/runtime/runtime-state.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.cc b/be/src/runtime/runtime-state.cc
index 40a4946..37d5fb0 100644
--- a/be/src/runtime/runtime-state.cc
+++ b/be/src/runtime/runtime-state.cc
@@ -96,16 +96,16 @@ RuntimeState::RuntimeState(const TQueryCtx& query_ctx)
RuntimeState::~RuntimeState() {
block_mgr_.reset();
+ // Release codegen memory before tearing down trackers.
+ codegen_.reset();
+
// query_mem_tracker_ must be valid as long as instance_mem_tracker_ is so
// delete instance_mem_tracker_ first.
// LogUsage() walks the MemTracker tree top-down when the memory limit is exceeded.
// Break the link between the instance_mem_tracker and its parent (query_mem_tracker_)
// before the instance_mem_tracker_ and its children are destroyed.
- if (instance_mem_tracker_.get() != NULL) {
- // May be NULL if InitMemTrackers() is not called, for example from tests.
- instance_mem_tracker_->UnregisterFromParent();
- }
-
+ // May be NULL if InitMemTrackers() is not called, for example from tests.
+ if (instance_mem_tracker_ != NULL) instance_mem_tracker_->UnregisterFromParent();
instance_mem_tracker_.reset();
query_mem_tracker_.reset();
}
@@ -184,8 +184,8 @@ Status RuntimeState::CreateBlockMgr() {
Status RuntimeState::CreateCodegen() {
if (codegen_.get() != NULL) return Status::OK();
// TODO: add the fragment ID to the codegen ID as well
- RETURN_IF_ERROR(LlvmCodeGen::CreateImpalaCodegen(
- obj_pool_.get(), PrintId(fragment_instance_id()), &codegen_));
+ RETURN_IF_ERROR(LlvmCodeGen::CreateImpalaCodegen(obj_pool_.get(),
+ instance_mem_tracker_.get(), PrintId(fragment_instance_id()), &codegen_));
codegen_->EnableOptimizations(true);
profile_.AddChild(codegen_->runtime_profile());
return Status::OK();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/common/thrift/PlanNodes.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/PlanNodes.thrift b/common/thrift/PlanNodes.thrift
index 6f863b2..233fbb2 100644
--- a/common/thrift/PlanNodes.thrift
+++ b/common/thrift/PlanNodes.thrift
@@ -463,32 +463,36 @@ struct TPlanNode {
6: required list<bool> nullable_tuples
7: optional list<Exprs.TExpr> conjuncts
+ // Set to true if codegen should be disabled for this plan node. Otherwise the plan
+ // node is codegen'd if the backend supports it.
+ 8: required bool disable_codegen
+
// one field per PlanNode subclass
- 8: optional THdfsScanNode hdfs_scan_node
- 9: optional THBaseScanNode hbase_scan_node
- 23: optional TKuduScanNode kudu_scan_node
- 10: optional TDataSourceScanNode data_source_node
- 11: optional THashJoinNode hash_join_node
- 12: optional TNestedLoopJoinNode nested_loop_join_node
- 13: optional TAggregationNode agg_node
- 14: optional TSortNode sort_node
- 15: optional TUnionNode union_node
- 16: optional TExchangeNode exchange_node
- 17: optional TAnalyticNode analytic_node
- 21: optional TUnnestNode unnest_node
+ 9: optional THdfsScanNode hdfs_scan_node
+ 10: optional THBaseScanNode hbase_scan_node
+ 11: optional TKuduScanNode kudu_scan_node
+ 12: optional TDataSourceScanNode data_source_node
+ 13: optional THashJoinNode hash_join_node
+ 14: optional TNestedLoopJoinNode nested_loop_join_node
+ 15: optional TAggregationNode agg_node
+ 16: optional TSortNode sort_node
+ 17: optional TUnionNode union_node
+ 18: optional TExchangeNode exchange_node
+ 19: optional TAnalyticNode analytic_node
+ 20: optional TUnnestNode unnest_node
// Label that should be used to print this node to the user.
- 18: optional string label
+ 21: optional string label
// Additional details that should be printed to the user. This is node specific
// e.g. table name, join strategy, etc.
- 19: optional string label_detail
+ 22: optional string label_detail
// Estimated execution stats generated by the planner.
- 20: optional ExecStats.TExecStats estimated_stats
+ 23: optional ExecStats.TExecStats estimated_stats
// Runtime filters assigned to this plan node
- 22: optional list<TRuntimeFilterDesc> runtime_filters
+ 24: optional list<TRuntimeFilterDesc> runtime_filters
}
// A flattened representation of a tree of PlanNodes, obtained by depth-first
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java b/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
index 620ea56..24d7caa 100644
--- a/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
@@ -818,6 +818,12 @@ public class DistributedPlanner {
mergeFragment.getPlanRoot(), node.getAggInfo().getMergeAggInfo());
mergeAggNode.init(ctx_.getRootAnalyzer());
mergeAggNode.setLimit(limit);
+ // Merge of non-grouping agg only processes one tuple per Impala daemon - codegen
+ // will cost more than benefit.
+ if (!hasGrouping) {
+ mergeFragment.getPlanRoot().setDisableCodegen(true);
+ mergeAggNode.setDisableCodegen(true);
+ }
// HAVING predicates can only be evaluated after the merge agg step
node.transferConjuncts(mergeAggNode);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/fe/src/main/java/org/apache/impala/planner/PlanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/PlanNode.java b/fe/src/main/java/org/apache/impala/planner/PlanNode.java
index 141464e..312ea5b 100644
--- a/fe/src/main/java/org/apache/impala/planner/PlanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/PlanNode.java
@@ -120,6 +120,9 @@ abstract public class PlanNode extends TreeNode<PlanNode> {
// set in computeCosts(); invalid: -1
protected long perHostMemCost_ = -1;
+ // If true, disable codegen for this plan node.
+ protected boolean disableCodegen_;
+
// Runtime filters assigned to this node.
protected List<RuntimeFilter> runtimeFilters_ = Lists.newArrayList();
@@ -144,6 +147,7 @@ abstract public class PlanNode extends TreeNode<PlanNode> {
cardinality_ = -1;
numNodes_ = -1;
displayName_ = displayName;
+ disableCodegen_ = false;
}
/**
@@ -159,6 +163,7 @@ abstract public class PlanNode extends TreeNode<PlanNode> {
cardinality_ = -1;
numNodes_ = -1;
displayName_ = displayName;
+ disableCodegen_ = node.disableCodegen_;
}
/**
@@ -393,9 +398,10 @@ abstract public class PlanNode extends TreeNode<PlanNode> {
msg.addToConjuncts(e.treeToThrift());
}
// Serialize any runtime filters
- for (RuntimeFilter filter: runtimeFilters_) {
+ for (RuntimeFilter filter : runtimeFilters_) {
msg.addToRuntime_filters(filter.toThrift());
}
+ msg.setDisable_codegen(disableCodegen_);
toThrift(msg);
container.addToNodes(msg);
// For the purpose of the BE consider ExchangeNodes to have no children.
@@ -718,4 +724,8 @@ abstract public class PlanNode extends TreeNode<PlanNode> {
return sortedConjuncts;
}
+
+ public void setDisableCodegen(boolean disableCodegen) {
+ disableCodegen_ = disableCodegen;
+ }
}
[3/6] incubator-impala git commit: IMPALA-3342: Add thread counters
to monitor plan fragment execution
Posted by kw...@apache.org.
IMPALA-3342: Add thread counters to monitor plan fragment execution
This change removes the use of total_cpu_timer which incorrectly
monitors the CPU time. Adding THREAD_COUNTERS to measure the user
and sys time in plan fragment execution. This also accounts for the
time spent in the hdfs/kudu scanner and in a blocking join.
Snippet of a query plan with the newly added PlanFragment
THREAD_COUNTERS:
Instance 2b40b101e2626e7a:a3d8f2300000000
- PeakMemoryUsage: 32.02 KB (32784)
- PerHostPeakMemUsage: 430.52 MB (451431312)
- RowsProduced: 1 (1)
- TotalNetworkReceiveTime: 10s379ms
- TotalNetworkSendTime: 0.000ns
- TotalStorageWaitTime: 0.000ns
- TotalWallClockTime: 10s577ms
- SysTime: 8.000ms
- UserTime: 8.000ms
- VoluntaryContextSwitches: 80 (80)
Change-Id: Ifa88aa6f3371fa42d11ecc122f43c7d83623c300
Reviewed-on: http://gerrit.cloudera.org:8080/4633
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/bb1c6338
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/bb1c6338
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/bb1c6338
Branch: refs/heads/master
Commit: bb1c63380b8ec14fb6058f3157cad8746463e054
Parents: 8f2bb2f
Author: aphadke <ap...@cloudera.com>
Authored: Thu Oct 6 22:10:51 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Wed Nov 23 00:56:48 2016 +0000
----------------------------------------------------------------------
be/src/exec/blocking-join-node.cc | 2 +-
be/src/exec/hdfs-scan-node.cc | 2 +-
be/src/exec/kudu-scan-node.cc | 3 +--
be/src/runtime/plan-fragment-executor.cc | 16 +++-------------
be/src/runtime/plan-fragment-executor.h | 3 ---
be/src/runtime/runtime-state.cc | 2 +-
be/src/runtime/runtime-state.h | 14 ++++++++++----
7 files changed, 17 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb1c6338/be/src/exec/blocking-join-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/blocking-join-node.cc b/be/src/exec/blocking-join-node.cc
index b114451..0e23727 100644
--- a/be/src/exec/blocking-join-node.cc
+++ b/be/src/exec/blocking-join-node.cc
@@ -145,9 +145,9 @@ void BlockingJoinNode::Close(RuntimeState* state) {
void BlockingJoinNode::ProcessBuildInputAsync(RuntimeState* state, DataSink* build_sink,
Promise<Status>* status) {
+ SCOPED_THREAD_COUNTER_MEASUREMENT(state->total_thread_statistics());
Status s;
{
- SCOPED_TIMER(state->total_cpu_timer());
if (build_sink == NULL){
s = ProcessBuildInput(state);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb1c6338/be/src/exec/hdfs-scan-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node.cc b/be/src/exec/hdfs-scan-node.cc
index 3798107..eebf075 100644
--- a/be/src/exec/hdfs-scan-node.cc
+++ b/be/src/exec/hdfs-scan-node.cc
@@ -356,7 +356,7 @@ void HdfsScanNode::ThreadTokenAvailableCb(ThreadResourceMgr::ResourcePool* pool)
void HdfsScanNode::ScannerThread() {
SCOPED_THREAD_COUNTER_MEASUREMENT(scanner_thread_counters());
- SCOPED_TIMER(runtime_state_->total_cpu_timer());
+ SCOPED_THREAD_COUNTER_MEASUREMENT(runtime_state_->total_thread_statistics());
// Make thread-local copy of filter contexts to prune scan ranges, and to pass to the
// scanner for finer-grained filtering.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb1c6338/be/src/exec/kudu-scan-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-scan-node.cc b/be/src/exec/kudu-scan-node.cc
index f98077e..40689ee 100644
--- a/be/src/exec/kudu-scan-node.cc
+++ b/be/src/exec/kudu-scan-node.cc
@@ -273,8 +273,7 @@ Status KuduScanNode::ProcessScanToken(KuduScanner* scanner, const string& scan_t
void KuduScanNode::RunScannerThread(const string& name, const string* initial_token) {
DCHECK(initial_token != NULL);
SCOPED_THREAD_COUNTER_MEASUREMENT(scanner_thread_counters());
- SCOPED_TIMER(runtime_state_->total_cpu_timer());
-
+ SCOPED_THREAD_COUNTER_MEASUREMENT(runtime_state_->total_thread_statistics());
// Set to true if this thread observes that the number of optional threads has been
// exceeded and is exiting early.
bool optional_thread_exiting = false;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb1c6338/be/src/runtime/plan-fragment-executor.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/plan-fragment-executor.cc b/be/src/runtime/plan-fragment-executor.cc
index 9db4d38..f673f34 100644
--- a/be/src/runtime/plan-fragment-executor.cc
+++ b/be/src/runtime/plan-fragment-executor.cc
@@ -39,6 +39,7 @@
#include "runtime/row-batch.h"
#include "runtime/runtime-filter-bank.h"
#include "util/container-util.h"
+#include "runtime/runtime-state.h"
#include "util/cpu-info.h"
#include "util/debug-util.h"
#include "util/mem-info.h"
@@ -107,7 +108,6 @@ Status PlanFragmentExecutor::PrepareInternal(const TExecPlanFragmentParams& requ
is_prepared_ = true;
// TODO: Break this method up.
- fragment_sw_.Start();
const TPlanFragmentInstanceCtx& fragment_instance_ctx = request.fragment_instance_ctx;
query_id_ = request.query_ctx.query_id;
@@ -302,6 +302,7 @@ Status PlanFragmentExecutor::Open() {
}
Status PlanFragmentExecutor::OpenInternal() {
+ SCOPED_THREAD_COUNTER_MEASUREMENT(runtime_state_->total_thread_statistics());
RETURN_IF_ERROR(
runtime_state_->desc_tbl().PrepareAndOpenPartitionExprs(runtime_state_.get()));
@@ -357,6 +358,7 @@ Status PlanFragmentExecutor::Exec() {
Status PlanFragmentExecutor::ExecInternal() {
RuntimeProfile::Counter* plan_exec_timer =
ADD_CHILD_TIMER(timings_profile_, "ExecTreeExecTime", EXEC_TIMER_NAME);
+ SCOPED_THREAD_COUNTER_MEASUREMENT(runtime_state_->total_thread_statistics());
bool exec_tree_complete = false;
do {
Status status;
@@ -467,18 +469,6 @@ void PlanFragmentExecutor::FragmentComplete() {
// Check the atomic flag. If it is set, then a fragment complete report has already
// been sent.
bool send_report = completed_report_sent_.CompareAndSwap(0, 1);
-
- fragment_sw_.Stop();
- int64_t cpu_and_wait_time = fragment_sw_.ElapsedTime();
- fragment_sw_ = MonotonicStopWatch();
- int64_t cpu_time = cpu_and_wait_time
- - runtime_state_->total_storage_wait_timer()->value()
- - runtime_state_->total_network_send_timer()->value()
- - runtime_state_->total_network_receive_timer()->value();
- // Timing is not perfect.
- if (cpu_time < 0) cpu_time = 0;
- runtime_state_->total_cpu_timer()->Add(cpu_time);
-
ReleaseThreadToken();
StopReportThread();
if (send_report) SendReport(true);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb1c6338/be/src/runtime/plan-fragment-executor.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/plan-fragment-executor.h b/be/src/runtime/plan-fragment-executor.h
index 7149272..708e979 100644
--- a/be/src/runtime/plan-fragment-executor.h
+++ b/be/src/runtime/plan-fragment-executor.h
@@ -242,9 +242,6 @@ class PlanFragmentExecutor {
/// of the execution.
RuntimeProfile::Counter* average_thread_tokens_;
- /// Stopwatch for this entire fragment. Started in Prepare(), stopped in Close().
- MonotonicStopWatch fragment_sw_;
-
/// (Atomic) Flag that indicates whether a completed fragment report has been or will
/// be fired. It is initialized to 0 and atomically swapped to 1 when a completed
/// fragment report is about to be fired. Used for reducing the probability that a
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb1c6338/be/src/runtime/runtime-state.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.cc b/be/src/runtime/runtime-state.cc
index c2adcb7..40a4946 100644
--- a/be/src/runtime/runtime-state.cc
+++ b/be/src/runtime/runtime-state.cc
@@ -134,7 +134,7 @@ Status RuntimeState::Init(ExecEnv* exec_env) {
DCHECK(resource_pool_ != NULL);
}
- total_cpu_timer_ = ADD_TIMER(runtime_profile(), "TotalCpuTime");
+ total_thread_statistics_ = ADD_THREAD_COUNTERS(runtime_profile(), "");
total_storage_wait_timer_ = ADD_TIMER(runtime_profile(), "TotalStorageWaitTime");
total_network_send_timer_ = ADD_TIMER(runtime_profile(), "TotalNetworkSendTime");
total_network_receive_timer_ = ADD_TIMER(runtime_profile(), "TotalNetworkReceiveTime");
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bb1c6338/be/src/runtime/runtime-state.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.h b/be/src/runtime/runtime-state.h
index 0ada7e4..f86374e 100644
--- a/be/src/runtime/runtime-state.h
+++ b/be/src/runtime/runtime-state.h
@@ -251,17 +251,22 @@ class RuntimeState {
bool is_cancelled() const { return is_cancelled_; }
void set_is_cancelled(bool v) { is_cancelled_ = v; }
- RuntimeProfile::Counter* total_cpu_timer() { return total_cpu_timer_; }
RuntimeProfile::Counter* total_storage_wait_timer() {
return total_storage_wait_timer_;
}
+
RuntimeProfile::Counter* total_network_send_timer() {
return total_network_send_timer_;
}
+
RuntimeProfile::Counter* total_network_receive_timer() {
return total_network_receive_timer_;
}
+ RuntimeProfile::ThreadCounters* total_thread_statistics() const {
+ return total_thread_statistics_;
+ }
+
/// Sets query_status_ with err_msg if no error has been set yet.
void SetQueryStatus(const std::string& err_msg) {
boost::lock_guard<SpinLock> l(query_status_lock_);
@@ -351,9 +356,6 @@ class RuntimeState {
RuntimeProfile profile_;
- /// Total CPU time (across all threads), including all wait times.
- RuntimeProfile::Counter* total_cpu_timer_;
-
/// Total time waiting in storage (across all threads)
RuntimeProfile::Counter* total_storage_wait_timer_;
@@ -363,6 +365,9 @@ class RuntimeState {
/// Total time spent receiving over the network (across all threads)
RuntimeProfile::Counter* total_network_receive_timer_;
+ /// Total CPU utilization for all threads in this plan fragment.
+ RuntimeProfile::ThreadCounters* total_thread_statistics_;
+
/// MemTracker that is shared by all fragment instances running on this host.
/// The query mem tracker must be released after the instance_mem_tracker_.
std::shared_ptr<MemTracker> query_mem_tracker_;
@@ -405,6 +410,7 @@ class RuntimeState {
/// prohibit copies
RuntimeState(const RuntimeState&);
+
};
#define RETURN_IF_CANCELLED(state) \
[5/6] incubator-impala git commit: IMPALA-4397,
IMPALA-3259: reduce codegen time and memory
Posted by kw...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test b/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
index 817dd4d..b741c5a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
@@ -988,3 +988,1019 @@ show column stats alltypes_no_coldef
---- TYPES
STRING, STRING, BIGINT, BIGINT, INT, DOUBLE
====
+---- QUERY
+# Test that compute stats works on wide tables.
+create table widetable_1000_cols
+stored as parquet
+as
+select * from functional_parquet.widetable_1000_cols
+====
+---- QUERY
+compute stats widetable_1000_cols
+====
+---- QUERY
+show column stats widetable_1000_cols
+---- RESULTS
+'bool_col1','BOOLEAN',2,-1,1,1
+'tinyint_col1','TINYINT',5,-1,1,1
+'smallint_col1','SMALLINT',5,-1,2,2
+'int_col1','INT',5,-1,4,4
+'bigint_col1','BIGINT',5,-1,8,8
+'float_col1','FLOAT',5,-1,4,4
+'double_col1','DOUBLE',5,-1,8,8
+'string_col1','STRING',5,-1,1,1
+'bool_col2','BOOLEAN',2,-1,1,1
+'tinyint_col2','TINYINT',5,-1,1,1
+'smallint_col2','SMALLINT',5,-1,2,2
+'int_col2','INT',5,-1,4,4
+'bigint_col2','BIGINT',5,-1,8,8
+'float_col2','FLOAT',5,-1,4,4
+'double_col2','DOUBLE',5,-1,8,8
+'string_col2','STRING',5,-1,1,1
+'bool_col3','BOOLEAN',2,-1,1,1
+'tinyint_col3','TINYINT',5,-1,1,1
+'smallint_col3','SMALLINT',5,-1,2,2
+'int_col3','INT',5,-1,4,4
+'bigint_col3','BIGINT',5,-1,8,8
+'float_col3','FLOAT',5,-1,4,4
+'double_col3','DOUBLE',5,-1,8,8
+'string_col3','STRING',5,-1,1,1
+'bool_col4','BOOLEAN',2,-1,1,1
+'tinyint_col4','TINYINT',5,-1,1,1
+'smallint_col4','SMALLINT',5,-1,2,2
+'int_col4','INT',5,-1,4,4
+'bigint_col4','BIGINT',5,-1,8,8
+'float_col4','FLOAT',5,-1,4,4
+'double_col4','DOUBLE',5,-1,8,8
+'string_col4','STRING',5,-1,1,1
+'bool_col5','BOOLEAN',2,-1,1,1
+'tinyint_col5','TINYINT',5,-1,1,1
+'smallint_col5','SMALLINT',5,-1,2,2
+'int_col5','INT',5,-1,4,4
+'bigint_col5','BIGINT',5,-1,8,8
+'float_col5','FLOAT',5,-1,4,4
+'double_col5','DOUBLE',5,-1,8,8
+'string_col5','STRING',5,-1,1,1
+'bool_col6','BOOLEAN',2,-1,1,1
+'tinyint_col6','TINYINT',5,-1,1,1
+'smallint_col6','SMALLINT',5,-1,2,2
+'int_col6','INT',5,-1,4,4
+'bigint_col6','BIGINT',5,-1,8,8
+'float_col6','FLOAT',5,-1,4,4
+'double_col6','DOUBLE',5,-1,8,8
+'string_col6','STRING',5,-1,1,1
+'bool_col7','BOOLEAN',2,-1,1,1
+'tinyint_col7','TINYINT',5,-1,1,1
+'smallint_col7','SMALLINT',5,-1,2,2
+'int_col7','INT',5,-1,4,4
+'bigint_col7','BIGINT',5,-1,8,8
+'float_col7','FLOAT',5,-1,4,4
+'double_col7','DOUBLE',5,-1,8,8
+'string_col7','STRING',5,-1,1,1
+'bool_col8','BOOLEAN',2,-1,1,1
+'tinyint_col8','TINYINT',5,-1,1,1
+'smallint_col8','SMALLINT',5,-1,2,2
+'int_col8','INT',5,-1,4,4
+'bigint_col8','BIGINT',5,-1,8,8
+'float_col8','FLOAT',5,-1,4,4
+'double_col8','DOUBLE',5,-1,8,8
+'string_col8','STRING',5,-1,1,1
+'bool_col9','BOOLEAN',2,-1,1,1
+'tinyint_col9','TINYINT',5,-1,1,1
+'smallint_col9','SMALLINT',5,-1,2,2
+'int_col9','INT',5,-1,4,4
+'bigint_col9','BIGINT',5,-1,8,8
+'float_col9','FLOAT',5,-1,4,4
+'double_col9','DOUBLE',5,-1,8,8
+'string_col9','STRING',5,-1,1,1
+'bool_col10','BOOLEAN',2,-1,1,1
+'tinyint_col10','TINYINT',5,-1,1,1
+'smallint_col10','SMALLINT',5,-1,2,2
+'int_col10','INT',5,-1,4,4
+'bigint_col10','BIGINT',5,-1,8,8
+'float_col10','FLOAT',5,-1,4,4
+'double_col10','DOUBLE',5,-1,8,8
+'string_col10','STRING',5,-1,1,1
+'bool_col11','BOOLEAN',2,-1,1,1
+'tinyint_col11','TINYINT',5,-1,1,1
+'smallint_col11','SMALLINT',5,-1,2,2
+'int_col11','INT',5,-1,4,4
+'bigint_col11','BIGINT',5,-1,8,8
+'float_col11','FLOAT',5,-1,4,4
+'double_col11','DOUBLE',5,-1,8,8
+'string_col11','STRING',5,-1,1,1
+'bool_col12','BOOLEAN',2,-1,1,1
+'tinyint_col12','TINYINT',5,-1,1,1
+'smallint_col12','SMALLINT',5,-1,2,2
+'int_col12','INT',5,-1,4,4
+'bigint_col12','BIGINT',5,-1,8,8
+'float_col12','FLOAT',5,-1,4,4
+'double_col12','DOUBLE',5,-1,8,8
+'string_col12','STRING',5,-1,1,1
+'bool_col13','BOOLEAN',2,-1,1,1
+'tinyint_col13','TINYINT',5,-1,1,1
+'smallint_col13','SMALLINT',5,-1,2,2
+'int_col13','INT',5,-1,4,4
+'bigint_col13','BIGINT',5,-1,8,8
+'float_col13','FLOAT',5,-1,4,4
+'double_col13','DOUBLE',5,-1,8,8
+'string_col13','STRING',5,-1,1,1
+'bool_col14','BOOLEAN',2,-1,1,1
+'tinyint_col14','TINYINT',5,-1,1,1
+'smallint_col14','SMALLINT',5,-1,2,2
+'int_col14','INT',5,-1,4,4
+'bigint_col14','BIGINT',5,-1,8,8
+'float_col14','FLOAT',5,-1,4,4
+'double_col14','DOUBLE',5,-1,8,8
+'string_col14','STRING',5,-1,1,1
+'bool_col15','BOOLEAN',2,-1,1,1
+'tinyint_col15','TINYINT',5,-1,1,1
+'smallint_col15','SMALLINT',5,-1,2,2
+'int_col15','INT',5,-1,4,4
+'bigint_col15','BIGINT',5,-1,8,8
+'float_col15','FLOAT',5,-1,4,4
+'double_col15','DOUBLE',5,-1,8,8
+'string_col15','STRING',5,-1,1,1
+'bool_col16','BOOLEAN',2,-1,1,1
+'tinyint_col16','TINYINT',5,-1,1,1
+'smallint_col16','SMALLINT',5,-1,2,2
+'int_col16','INT',5,-1,4,4
+'bigint_col16','BIGINT',5,-1,8,8
+'float_col16','FLOAT',5,-1,4,4
+'double_col16','DOUBLE',5,-1,8,8
+'string_col16','STRING',5,-1,1,1
+'bool_col17','BOOLEAN',2,-1,1,1
+'tinyint_col17','TINYINT',5,-1,1,1
+'smallint_col17','SMALLINT',5,-1,2,2
+'int_col17','INT',5,-1,4,4
+'bigint_col17','BIGINT',5,-1,8,8
+'float_col17','FLOAT',5,-1,4,4
+'double_col17','DOUBLE',5,-1,8,8
+'string_col17','STRING',5,-1,1,1
+'bool_col18','BOOLEAN',2,-1,1,1
+'tinyint_col18','TINYINT',5,-1,1,1
+'smallint_col18','SMALLINT',5,-1,2,2
+'int_col18','INT',5,-1,4,4
+'bigint_col18','BIGINT',5,-1,8,8
+'float_col18','FLOAT',5,-1,4,4
+'double_col18','DOUBLE',5,-1,8,8
+'string_col18','STRING',5,-1,1,1
+'bool_col19','BOOLEAN',2,-1,1,1
+'tinyint_col19','TINYINT',5,-1,1,1
+'smallint_col19','SMALLINT',5,-1,2,2
+'int_col19','INT',5,-1,4,4
+'bigint_col19','BIGINT',5,-1,8,8
+'float_col19','FLOAT',5,-1,4,4
+'double_col19','DOUBLE',5,-1,8,8
+'string_col19','STRING',5,-1,1,1
+'bool_col20','BOOLEAN',2,-1,1,1
+'tinyint_col20','TINYINT',5,-1,1,1
+'smallint_col20','SMALLINT',5,-1,2,2
+'int_col20','INT',5,-1,4,4
+'bigint_col20','BIGINT',5,-1,8,8
+'float_col20','FLOAT',5,-1,4,4
+'double_col20','DOUBLE',5,-1,8,8
+'string_col20','STRING',5,-1,1,1
+'bool_col21','BOOLEAN',2,-1,1,1
+'tinyint_col21','TINYINT',5,-1,1,1
+'smallint_col21','SMALLINT',5,-1,2,2
+'int_col21','INT',5,-1,4,4
+'bigint_col21','BIGINT',5,-1,8,8
+'float_col21','FLOAT',5,-1,4,4
+'double_col21','DOUBLE',5,-1,8,8
+'string_col21','STRING',5,-1,1,1
+'bool_col22','BOOLEAN',2,-1,1,1
+'tinyint_col22','TINYINT',5,-1,1,1
+'smallint_col22','SMALLINT',5,-1,2,2
+'int_col22','INT',5,-1,4,4
+'bigint_col22','BIGINT',5,-1,8,8
+'float_col22','FLOAT',5,-1,4,4
+'double_col22','DOUBLE',5,-1,8,8
+'string_col22','STRING',5,-1,1,1
+'bool_col23','BOOLEAN',2,-1,1,1
+'tinyint_col23','TINYINT',5,-1,1,1
+'smallint_col23','SMALLINT',5,-1,2,2
+'int_col23','INT',5,-1,4,4
+'bigint_col23','BIGINT',5,-1,8,8
+'float_col23','FLOAT',5,-1,4,4
+'double_col23','DOUBLE',5,-1,8,8
+'string_col23','STRING',5,-1,1,1
+'bool_col24','BOOLEAN',2,-1,1,1
+'tinyint_col24','TINYINT',5,-1,1,1
+'smallint_col24','SMALLINT',5,-1,2,2
+'int_col24','INT',5,-1,4,4
+'bigint_col24','BIGINT',5,-1,8,8
+'float_col24','FLOAT',5,-1,4,4
+'double_col24','DOUBLE',5,-1,8,8
+'string_col24','STRING',5,-1,1,1
+'bool_col25','BOOLEAN',2,-1,1,1
+'tinyint_col25','TINYINT',5,-1,1,1
+'smallint_col25','SMALLINT',5,-1,2,2
+'int_col25','INT',5,-1,4,4
+'bigint_col25','BIGINT',5,-1,8,8
+'float_col25','FLOAT',5,-1,4,4
+'double_col25','DOUBLE',5,-1,8,8
+'string_col25','STRING',5,-1,1,1
+'bool_col26','BOOLEAN',2,-1,1,1
+'tinyint_col26','TINYINT',5,-1,1,1
+'smallint_col26','SMALLINT',5,-1,2,2
+'int_col26','INT',5,-1,4,4
+'bigint_col26','BIGINT',5,-1,8,8
+'float_col26','FLOAT',5,-1,4,4
+'double_col26','DOUBLE',5,-1,8,8
+'string_col26','STRING',5,-1,1,1
+'bool_col27','BOOLEAN',2,-1,1,1
+'tinyint_col27','TINYINT',5,-1,1,1
+'smallint_col27','SMALLINT',5,-1,2,2
+'int_col27','INT',5,-1,4,4
+'bigint_col27','BIGINT',5,-1,8,8
+'float_col27','FLOAT',5,-1,4,4
+'double_col27','DOUBLE',5,-1,8,8
+'string_col27','STRING',5,-1,1,1
+'bool_col28','BOOLEAN',2,-1,1,1
+'tinyint_col28','TINYINT',5,-1,1,1
+'smallint_col28','SMALLINT',5,-1,2,2
+'int_col28','INT',5,-1,4,4
+'bigint_col28','BIGINT',5,-1,8,8
+'float_col28','FLOAT',5,-1,4,4
+'double_col28','DOUBLE',5,-1,8,8
+'string_col28','STRING',5,-1,1,1
+'bool_col29','BOOLEAN',2,-1,1,1
+'tinyint_col29','TINYINT',5,-1,1,1
+'smallint_col29','SMALLINT',5,-1,2,2
+'int_col29','INT',5,-1,4,4
+'bigint_col29','BIGINT',5,-1,8,8
+'float_col29','FLOAT',5,-1,4,4
+'double_col29','DOUBLE',5,-1,8,8
+'string_col29','STRING',5,-1,1,1
+'bool_col30','BOOLEAN',2,-1,1,1
+'tinyint_col30','TINYINT',5,-1,1,1
+'smallint_col30','SMALLINT',5,-1,2,2
+'int_col30','INT',5,-1,4,4
+'bigint_col30','BIGINT',5,-1,8,8
+'float_col30','FLOAT',5,-1,4,4
+'double_col30','DOUBLE',5,-1,8,8
+'string_col30','STRING',5,-1,1,1
+'bool_col31','BOOLEAN',2,-1,1,1
+'tinyint_col31','TINYINT',5,-1,1,1
+'smallint_col31','SMALLINT',5,-1,2,2
+'int_col31','INT',5,-1,4,4
+'bigint_col31','BIGINT',5,-1,8,8
+'float_col31','FLOAT',5,-1,4,4
+'double_col31','DOUBLE',5,-1,8,8
+'string_col31','STRING',5,-1,1,1
+'bool_col32','BOOLEAN',2,-1,1,1
+'tinyint_col32','TINYINT',5,-1,1,1
+'smallint_col32','SMALLINT',5,-1,2,2
+'int_col32','INT',5,-1,4,4
+'bigint_col32','BIGINT',5,-1,8,8
+'float_col32','FLOAT',5,-1,4,4
+'double_col32','DOUBLE',5,-1,8,8
+'string_col32','STRING',5,-1,1,1
+'bool_col33','BOOLEAN',2,-1,1,1
+'tinyint_col33','TINYINT',5,-1,1,1
+'smallint_col33','SMALLINT',5,-1,2,2
+'int_col33','INT',5,-1,4,4
+'bigint_col33','BIGINT',5,-1,8,8
+'float_col33','FLOAT',5,-1,4,4
+'double_col33','DOUBLE',5,-1,8,8
+'string_col33','STRING',5,-1,1,1
+'bool_col34','BOOLEAN',2,-1,1,1
+'tinyint_col34','TINYINT',5,-1,1,1
+'smallint_col34','SMALLINT',5,-1,2,2
+'int_col34','INT',5,-1,4,4
+'bigint_col34','BIGINT',5,-1,8,8
+'float_col34','FLOAT',5,-1,4,4
+'double_col34','DOUBLE',5,-1,8,8
+'string_col34','STRING',5,-1,1,1
+'bool_col35','BOOLEAN',2,-1,1,1
+'tinyint_col35','TINYINT',5,-1,1,1
+'smallint_col35','SMALLINT',5,-1,2,2
+'int_col35','INT',5,-1,4,4
+'bigint_col35','BIGINT',5,-1,8,8
+'float_col35','FLOAT',5,-1,4,4
+'double_col35','DOUBLE',5,-1,8,8
+'string_col35','STRING',5,-1,1,1
+'bool_col36','BOOLEAN',2,-1,1,1
+'tinyint_col36','TINYINT',5,-1,1,1
+'smallint_col36','SMALLINT',5,-1,2,2
+'int_col36','INT',5,-1,4,4
+'bigint_col36','BIGINT',5,-1,8,8
+'float_col36','FLOAT',5,-1,4,4
+'double_col36','DOUBLE',5,-1,8,8
+'string_col36','STRING',5,-1,1,1
+'bool_col37','BOOLEAN',2,-1,1,1
+'tinyint_col37','TINYINT',5,-1,1,1
+'smallint_col37','SMALLINT',5,-1,2,2
+'int_col37','INT',5,-1,4,4
+'bigint_col37','BIGINT',5,-1,8,8
+'float_col37','FLOAT',5,-1,4,4
+'double_col37','DOUBLE',5,-1,8,8
+'string_col37','STRING',5,-1,1,1
+'bool_col38','BOOLEAN',2,-1,1,1
+'tinyint_col38','TINYINT',5,-1,1,1
+'smallint_col38','SMALLINT',5,-1,2,2
+'int_col38','INT',5,-1,4,4
+'bigint_col38','BIGINT',5,-1,8,8
+'float_col38','FLOAT',5,-1,4,4
+'double_col38','DOUBLE',5,-1,8,8
+'string_col38','STRING',5,-1,1,1
+'bool_col39','BOOLEAN',2,-1,1,1
+'tinyint_col39','TINYINT',5,-1,1,1
+'smallint_col39','SMALLINT',5,-1,2,2
+'int_col39','INT',5,-1,4,4
+'bigint_col39','BIGINT',5,-1,8,8
+'float_col39','FLOAT',5,-1,4,4
+'double_col39','DOUBLE',5,-1,8,8
+'string_col39','STRING',5,-1,1,1
+'bool_col40','BOOLEAN',2,-1,1,1
+'tinyint_col40','TINYINT',5,-1,1,1
+'smallint_col40','SMALLINT',5,-1,2,2
+'int_col40','INT',5,-1,4,4
+'bigint_col40','BIGINT',5,-1,8,8
+'float_col40','FLOAT',5,-1,4,4
+'double_col40','DOUBLE',5,-1,8,8
+'string_col40','STRING',5,-1,1,1
+'bool_col41','BOOLEAN',2,-1,1,1
+'tinyint_col41','TINYINT',5,-1,1,1
+'smallint_col41','SMALLINT',5,-1,2,2
+'int_col41','INT',5,-1,4,4
+'bigint_col41','BIGINT',5,-1,8,8
+'float_col41','FLOAT',5,-1,4,4
+'double_col41','DOUBLE',5,-1,8,8
+'string_col41','STRING',5,-1,1,1
+'bool_col42','BOOLEAN',2,-1,1,1
+'tinyint_col42','TINYINT',5,-1,1,1
+'smallint_col42','SMALLINT',5,-1,2,2
+'int_col42','INT',5,-1,4,4
+'bigint_col42','BIGINT',5,-1,8,8
+'float_col42','FLOAT',5,-1,4,4
+'double_col42','DOUBLE',5,-1,8,8
+'string_col42','STRING',5,-1,1,1
+'bool_col43','BOOLEAN',2,-1,1,1
+'tinyint_col43','TINYINT',5,-1,1,1
+'smallint_col43','SMALLINT',5,-1,2,2
+'int_col43','INT',5,-1,4,4
+'bigint_col43','BIGINT',5,-1,8,8
+'float_col43','FLOAT',5,-1,4,4
+'double_col43','DOUBLE',5,-1,8,8
+'string_col43','STRING',5,-1,1,1
+'bool_col44','BOOLEAN',2,-1,1,1
+'tinyint_col44','TINYINT',5,-1,1,1
+'smallint_col44','SMALLINT',5,-1,2,2
+'int_col44','INT',5,-1,4,4
+'bigint_col44','BIGINT',5,-1,8,8
+'float_col44','FLOAT',5,-1,4,4
+'double_col44','DOUBLE',5,-1,8,8
+'string_col44','STRING',5,-1,1,1
+'bool_col45','BOOLEAN',2,-1,1,1
+'tinyint_col45','TINYINT',5,-1,1,1
+'smallint_col45','SMALLINT',5,-1,2,2
+'int_col45','INT',5,-1,4,4
+'bigint_col45','BIGINT',5,-1,8,8
+'float_col45','FLOAT',5,-1,4,4
+'double_col45','DOUBLE',5,-1,8,8
+'string_col45','STRING',5,-1,1,1
+'bool_col46','BOOLEAN',2,-1,1,1
+'tinyint_col46','TINYINT',5,-1,1,1
+'smallint_col46','SMALLINT',5,-1,2,2
+'int_col46','INT',5,-1,4,4
+'bigint_col46','BIGINT',5,-1,8,8
+'float_col46','FLOAT',5,-1,4,4
+'double_col46','DOUBLE',5,-1,8,8
+'string_col46','STRING',5,-1,1,1
+'bool_col47','BOOLEAN',2,-1,1,1
+'tinyint_col47','TINYINT',5,-1,1,1
+'smallint_col47','SMALLINT',5,-1,2,2
+'int_col47','INT',5,-1,4,4
+'bigint_col47','BIGINT',5,-1,8,8
+'float_col47','FLOAT',5,-1,4,4
+'double_col47','DOUBLE',5,-1,8,8
+'string_col47','STRING',5,-1,1,1
+'bool_col48','BOOLEAN',2,-1,1,1
+'tinyint_col48','TINYINT',5,-1,1,1
+'smallint_col48','SMALLINT',5,-1,2,2
+'int_col48','INT',5,-1,4,4
+'bigint_col48','BIGINT',5,-1,8,8
+'float_col48','FLOAT',5,-1,4,4
+'double_col48','DOUBLE',5,-1,8,8
+'string_col48','STRING',5,-1,1,1
+'bool_col49','BOOLEAN',2,-1,1,1
+'tinyint_col49','TINYINT',5,-1,1,1
+'smallint_col49','SMALLINT',5,-1,2,2
+'int_col49','INT',5,-1,4,4
+'bigint_col49','BIGINT',5,-1,8,8
+'float_col49','FLOAT',5,-1,4,4
+'double_col49','DOUBLE',5,-1,8,8
+'string_col49','STRING',5,-1,1,1
+'bool_col50','BOOLEAN',2,-1,1,1
+'tinyint_col50','TINYINT',5,-1,1,1
+'smallint_col50','SMALLINT',5,-1,2,2
+'int_col50','INT',5,-1,4,4
+'bigint_col50','BIGINT',5,-1,8,8
+'float_col50','FLOAT',5,-1,4,4
+'double_col50','DOUBLE',5,-1,8,8
+'string_col50','STRING',5,-1,1,1
+'bool_col51','BOOLEAN',2,-1,1,1
+'tinyint_col51','TINYINT',5,-1,1,1
+'smallint_col51','SMALLINT',5,-1,2,2
+'int_col51','INT',5,-1,4,4
+'bigint_col51','BIGINT',5,-1,8,8
+'float_col51','FLOAT',5,-1,4,4
+'double_col51','DOUBLE',5,-1,8,8
+'string_col51','STRING',5,-1,1,1
+'bool_col52','BOOLEAN',2,-1,1,1
+'tinyint_col52','TINYINT',5,-1,1,1
+'smallint_col52','SMALLINT',5,-1,2,2
+'int_col52','INT',5,-1,4,4
+'bigint_col52','BIGINT',5,-1,8,8
+'float_col52','FLOAT',5,-1,4,4
+'double_col52','DOUBLE',5,-1,8,8
+'string_col52','STRING',5,-1,1,1
+'bool_col53','BOOLEAN',2,-1,1,1
+'tinyint_col53','TINYINT',5,-1,1,1
+'smallint_col53','SMALLINT',5,-1,2,2
+'int_col53','INT',5,-1,4,4
+'bigint_col53','BIGINT',5,-1,8,8
+'float_col53','FLOAT',5,-1,4,4
+'double_col53','DOUBLE',5,-1,8,8
+'string_col53','STRING',5,-1,1,1
+'bool_col54','BOOLEAN',2,-1,1,1
+'tinyint_col54','TINYINT',5,-1,1,1
+'smallint_col54','SMALLINT',5,-1,2,2
+'int_col54','INT',5,-1,4,4
+'bigint_col54','BIGINT',5,-1,8,8
+'float_col54','FLOAT',5,-1,4,4
+'double_col54','DOUBLE',5,-1,8,8
+'string_col54','STRING',5,-1,1,1
+'bool_col55','BOOLEAN',2,-1,1,1
+'tinyint_col55','TINYINT',5,-1,1,1
+'smallint_col55','SMALLINT',5,-1,2,2
+'int_col55','INT',5,-1,4,4
+'bigint_col55','BIGINT',5,-1,8,8
+'float_col55','FLOAT',5,-1,4,4
+'double_col55','DOUBLE',5,-1,8,8
+'string_col55','STRING',5,-1,1,1
+'bool_col56','BOOLEAN',2,-1,1,1
+'tinyint_col56','TINYINT',5,-1,1,1
+'smallint_col56','SMALLINT',5,-1,2,2
+'int_col56','INT',5,-1,4,4
+'bigint_col56','BIGINT',5,-1,8,8
+'float_col56','FLOAT',5,-1,4,4
+'double_col56','DOUBLE',5,-1,8,8
+'string_col56','STRING',5,-1,1,1
+'bool_col57','BOOLEAN',2,-1,1,1
+'tinyint_col57','TINYINT',5,-1,1,1
+'smallint_col57','SMALLINT',5,-1,2,2
+'int_col57','INT',5,-1,4,4
+'bigint_col57','BIGINT',5,-1,8,8
+'float_col57','FLOAT',5,-1,4,4
+'double_col57','DOUBLE',5,-1,8,8
+'string_col57','STRING',5,-1,1,1
+'bool_col58','BOOLEAN',2,-1,1,1
+'tinyint_col58','TINYINT',5,-1,1,1
+'smallint_col58','SMALLINT',5,-1,2,2
+'int_col58','INT',5,-1,4,4
+'bigint_col58','BIGINT',5,-1,8,8
+'float_col58','FLOAT',5,-1,4,4
+'double_col58','DOUBLE',5,-1,8,8
+'string_col58','STRING',5,-1,1,1
+'bool_col59','BOOLEAN',2,-1,1,1
+'tinyint_col59','TINYINT',5,-1,1,1
+'smallint_col59','SMALLINT',5,-1,2,2
+'int_col59','INT',5,-1,4,4
+'bigint_col59','BIGINT',5,-1,8,8
+'float_col59','FLOAT',5,-1,4,4
+'double_col59','DOUBLE',5,-1,8,8
+'string_col59','STRING',5,-1,1,1
+'bool_col60','BOOLEAN',2,-1,1,1
+'tinyint_col60','TINYINT',5,-1,1,1
+'smallint_col60','SMALLINT',5,-1,2,2
+'int_col60','INT',5,-1,4,4
+'bigint_col60','BIGINT',5,-1,8,8
+'float_col60','FLOAT',5,-1,4,4
+'double_col60','DOUBLE',5,-1,8,8
+'string_col60','STRING',5,-1,1,1
+'bool_col61','BOOLEAN',2,-1,1,1
+'tinyint_col61','TINYINT',5,-1,1,1
+'smallint_col61','SMALLINT',5,-1,2,2
+'int_col61','INT',5,-1,4,4
+'bigint_col61','BIGINT',5,-1,8,8
+'float_col61','FLOAT',5,-1,4,4
+'double_col61','DOUBLE',5,-1,8,8
+'string_col61','STRING',5,-1,1,1
+'bool_col62','BOOLEAN',2,-1,1,1
+'tinyint_col62','TINYINT',5,-1,1,1
+'smallint_col62','SMALLINT',5,-1,2,2
+'int_col62','INT',5,-1,4,4
+'bigint_col62','BIGINT',5,-1,8,8
+'float_col62','FLOAT',5,-1,4,4
+'double_col62','DOUBLE',5,-1,8,8
+'string_col62','STRING',5,-1,1,1
+'bool_col63','BOOLEAN',2,-1,1,1
+'tinyint_col63','TINYINT',5,-1,1,1
+'smallint_col63','SMALLINT',5,-1,2,2
+'int_col63','INT',5,-1,4,4
+'bigint_col63','BIGINT',5,-1,8,8
+'float_col63','FLOAT',5,-1,4,4
+'double_col63','DOUBLE',5,-1,8,8
+'string_col63','STRING',5,-1,1,1
+'bool_col64','BOOLEAN',2,-1,1,1
+'tinyint_col64','TINYINT',5,-1,1,1
+'smallint_col64','SMALLINT',5,-1,2,2
+'int_col64','INT',5,-1,4,4
+'bigint_col64','BIGINT',5,-1,8,8
+'float_col64','FLOAT',5,-1,4,4
+'double_col64','DOUBLE',5,-1,8,8
+'string_col64','STRING',5,-1,1,1
+'bool_col65','BOOLEAN',2,-1,1,1
+'tinyint_col65','TINYINT',5,-1,1,1
+'smallint_col65','SMALLINT',5,-1,2,2
+'int_col65','INT',5,-1,4,4
+'bigint_col65','BIGINT',5,-1,8,8
+'float_col65','FLOAT',5,-1,4,4
+'double_col65','DOUBLE',5,-1,8,8
+'string_col65','STRING',5,-1,1,1
+'bool_col66','BOOLEAN',2,-1,1,1
+'tinyint_col66','TINYINT',5,-1,1,1
+'smallint_col66','SMALLINT',5,-1,2,2
+'int_col66','INT',5,-1,4,4
+'bigint_col66','BIGINT',5,-1,8,8
+'float_col66','FLOAT',5,-1,4,4
+'double_col66','DOUBLE',5,-1,8,8
+'string_col66','STRING',5,-1,1,1
+'bool_col67','BOOLEAN',2,-1,1,1
+'tinyint_col67','TINYINT',5,-1,1,1
+'smallint_col67','SMALLINT',5,-1,2,2
+'int_col67','INT',5,-1,4,4
+'bigint_col67','BIGINT',5,-1,8,8
+'float_col67','FLOAT',5,-1,4,4
+'double_col67','DOUBLE',5,-1,8,8
+'string_col67','STRING',5,-1,1,1
+'bool_col68','BOOLEAN',2,-1,1,1
+'tinyint_col68','TINYINT',5,-1,1,1
+'smallint_col68','SMALLINT',5,-1,2,2
+'int_col68','INT',5,-1,4,4
+'bigint_col68','BIGINT',5,-1,8,8
+'float_col68','FLOAT',5,-1,4,4
+'double_col68','DOUBLE',5,-1,8,8
+'string_col68','STRING',5,-1,1,1
+'bool_col69','BOOLEAN',2,-1,1,1
+'tinyint_col69','TINYINT',5,-1,1,1
+'smallint_col69','SMALLINT',5,-1,2,2
+'int_col69','INT',5,-1,4,4
+'bigint_col69','BIGINT',5,-1,8,8
+'float_col69','FLOAT',5,-1,4,4
+'double_col69','DOUBLE',5,-1,8,8
+'string_col69','STRING',5,-1,1,1
+'bool_col70','BOOLEAN',2,-1,1,1
+'tinyint_col70','TINYINT',5,-1,1,1
+'smallint_col70','SMALLINT',5,-1,2,2
+'int_col70','INT',5,-1,4,4
+'bigint_col70','BIGINT',5,-1,8,8
+'float_col70','FLOAT',5,-1,4,4
+'double_col70','DOUBLE',5,-1,8,8
+'string_col70','STRING',5,-1,1,1
+'bool_col71','BOOLEAN',2,-1,1,1
+'tinyint_col71','TINYINT',5,-1,1,1
+'smallint_col71','SMALLINT',5,-1,2,2
+'int_col71','INT',5,-1,4,4
+'bigint_col71','BIGINT',5,-1,8,8
+'float_col71','FLOAT',5,-1,4,4
+'double_col71','DOUBLE',5,-1,8,8
+'string_col71','STRING',5,-1,1,1
+'bool_col72','BOOLEAN',2,-1,1,1
+'tinyint_col72','TINYINT',5,-1,1,1
+'smallint_col72','SMALLINT',5,-1,2,2
+'int_col72','INT',5,-1,4,4
+'bigint_col72','BIGINT',5,-1,8,8
+'float_col72','FLOAT',5,-1,4,4
+'double_col72','DOUBLE',5,-1,8,8
+'string_col72','STRING',5,-1,1,1
+'bool_col73','BOOLEAN',2,-1,1,1
+'tinyint_col73','TINYINT',5,-1,1,1
+'smallint_col73','SMALLINT',5,-1,2,2
+'int_col73','INT',5,-1,4,4
+'bigint_col73','BIGINT',5,-1,8,8
+'float_col73','FLOAT',5,-1,4,4
+'double_col73','DOUBLE',5,-1,8,8
+'string_col73','STRING',5,-1,1,1
+'bool_col74','BOOLEAN',2,-1,1,1
+'tinyint_col74','TINYINT',5,-1,1,1
+'smallint_col74','SMALLINT',5,-1,2,2
+'int_col74','INT',5,-1,4,4
+'bigint_col74','BIGINT',5,-1,8,8
+'float_col74','FLOAT',5,-1,4,4
+'double_col74','DOUBLE',5,-1,8,8
+'string_col74','STRING',5,-1,1,1
+'bool_col75','BOOLEAN',2,-1,1,1
+'tinyint_col75','TINYINT',5,-1,1,1
+'smallint_col75','SMALLINT',5,-1,2,2
+'int_col75','INT',5,-1,4,4
+'bigint_col75','BIGINT',5,-1,8,8
+'float_col75','FLOAT',5,-1,4,4
+'double_col75','DOUBLE',5,-1,8,8
+'string_col75','STRING',5,-1,1,1
+'bool_col76','BOOLEAN',2,-1,1,1
+'tinyint_col76','TINYINT',5,-1,1,1
+'smallint_col76','SMALLINT',5,-1,2,2
+'int_col76','INT',5,-1,4,4
+'bigint_col76','BIGINT',5,-1,8,8
+'float_col76','FLOAT',5,-1,4,4
+'double_col76','DOUBLE',5,-1,8,8
+'string_col76','STRING',5,-1,1,1
+'bool_col77','BOOLEAN',2,-1,1,1
+'tinyint_col77','TINYINT',5,-1,1,1
+'smallint_col77','SMALLINT',5,-1,2,2
+'int_col77','INT',5,-1,4,4
+'bigint_col77','BIGINT',5,-1,8,8
+'float_col77','FLOAT',5,-1,4,4
+'double_col77','DOUBLE',5,-1,8,8
+'string_col77','STRING',5,-1,1,1
+'bool_col78','BOOLEAN',2,-1,1,1
+'tinyint_col78','TINYINT',5,-1,1,1
+'smallint_col78','SMALLINT',5,-1,2,2
+'int_col78','INT',5,-1,4,4
+'bigint_col78','BIGINT',5,-1,8,8
+'float_col78','FLOAT',5,-1,4,4
+'double_col78','DOUBLE',5,-1,8,8
+'string_col78','STRING',5,-1,1,1
+'bool_col79','BOOLEAN',2,-1,1,1
+'tinyint_col79','TINYINT',5,-1,1,1
+'smallint_col79','SMALLINT',5,-1,2,2
+'int_col79','INT',5,-1,4,4
+'bigint_col79','BIGINT',5,-1,8,8
+'float_col79','FLOAT',5,-1,4,4
+'double_col79','DOUBLE',5,-1,8,8
+'string_col79','STRING',5,-1,1,1
+'bool_col80','BOOLEAN',2,-1,1,1
+'tinyint_col80','TINYINT',5,-1,1,1
+'smallint_col80','SMALLINT',5,-1,2,2
+'int_col80','INT',5,-1,4,4
+'bigint_col80','BIGINT',5,-1,8,8
+'float_col80','FLOAT',5,-1,4,4
+'double_col80','DOUBLE',5,-1,8,8
+'string_col80','STRING',5,-1,1,1
+'bool_col81','BOOLEAN',2,-1,1,1
+'tinyint_col81','TINYINT',5,-1,1,1
+'smallint_col81','SMALLINT',5,-1,2,2
+'int_col81','INT',5,-1,4,4
+'bigint_col81','BIGINT',5,-1,8,8
+'float_col81','FLOAT',5,-1,4,4
+'double_col81','DOUBLE',5,-1,8,8
+'string_col81','STRING',5,-1,1,1
+'bool_col82','BOOLEAN',2,-1,1,1
+'tinyint_col82','TINYINT',5,-1,1,1
+'smallint_col82','SMALLINT',5,-1,2,2
+'int_col82','INT',5,-1,4,4
+'bigint_col82','BIGINT',5,-1,8,8
+'float_col82','FLOAT',5,-1,4,4
+'double_col82','DOUBLE',5,-1,8,8
+'string_col82','STRING',5,-1,1,1
+'bool_col83','BOOLEAN',2,-1,1,1
+'tinyint_col83','TINYINT',5,-1,1,1
+'smallint_col83','SMALLINT',5,-1,2,2
+'int_col83','INT',5,-1,4,4
+'bigint_col83','BIGINT',5,-1,8,8
+'float_col83','FLOAT',5,-1,4,4
+'double_col83','DOUBLE',5,-1,8,8
+'string_col83','STRING',5,-1,1,1
+'bool_col84','BOOLEAN',2,-1,1,1
+'tinyint_col84','TINYINT',5,-1,1,1
+'smallint_col84','SMALLINT',5,-1,2,2
+'int_col84','INT',5,-1,4,4
+'bigint_col84','BIGINT',5,-1,8,8
+'float_col84','FLOAT',5,-1,4,4
+'double_col84','DOUBLE',5,-1,8,8
+'string_col84','STRING',5,-1,1,1
+'bool_col85','BOOLEAN',2,-1,1,1
+'tinyint_col85','TINYINT',5,-1,1,1
+'smallint_col85','SMALLINT',5,-1,2,2
+'int_col85','INT',5,-1,4,4
+'bigint_col85','BIGINT',5,-1,8,8
+'float_col85','FLOAT',5,-1,4,4
+'double_col85','DOUBLE',5,-1,8,8
+'string_col85','STRING',5,-1,1,1
+'bool_col86','BOOLEAN',2,-1,1,1
+'tinyint_col86','TINYINT',5,-1,1,1
+'smallint_col86','SMALLINT',5,-1,2,2
+'int_col86','INT',5,-1,4,4
+'bigint_col86','BIGINT',5,-1,8,8
+'float_col86','FLOAT',5,-1,4,4
+'double_col86','DOUBLE',5,-1,8,8
+'string_col86','STRING',5,-1,1,1
+'bool_col87','BOOLEAN',2,-1,1,1
+'tinyint_col87','TINYINT',5,-1,1,1
+'smallint_col87','SMALLINT',5,-1,2,2
+'int_col87','INT',5,-1,4,4
+'bigint_col87','BIGINT',5,-1,8,8
+'float_col87','FLOAT',5,-1,4,4
+'double_col87','DOUBLE',5,-1,8,8
+'string_col87','STRING',5,-1,1,1
+'bool_col88','BOOLEAN',2,-1,1,1
+'tinyint_col88','TINYINT',5,-1,1,1
+'smallint_col88','SMALLINT',5,-1,2,2
+'int_col88','INT',5,-1,4,4
+'bigint_col88','BIGINT',5,-1,8,8
+'float_col88','FLOAT',5,-1,4,4
+'double_col88','DOUBLE',5,-1,8,8
+'string_col88','STRING',5,-1,1,1
+'bool_col89','BOOLEAN',2,-1,1,1
+'tinyint_col89','TINYINT',5,-1,1,1
+'smallint_col89','SMALLINT',5,-1,2,2
+'int_col89','INT',5,-1,4,4
+'bigint_col89','BIGINT',5,-1,8,8
+'float_col89','FLOAT',5,-1,4,4
+'double_col89','DOUBLE',5,-1,8,8
+'string_col89','STRING',5,-1,1,1
+'bool_col90','BOOLEAN',2,-1,1,1
+'tinyint_col90','TINYINT',5,-1,1,1
+'smallint_col90','SMALLINT',5,-1,2,2
+'int_col90','INT',5,-1,4,4
+'bigint_col90','BIGINT',5,-1,8,8
+'float_col90','FLOAT',5,-1,4,4
+'double_col90','DOUBLE',5,-1,8,8
+'string_col90','STRING',5,-1,1,1
+'bool_col91','BOOLEAN',2,-1,1,1
+'tinyint_col91','TINYINT',5,-1,1,1
+'smallint_col91','SMALLINT',5,-1,2,2
+'int_col91','INT',5,-1,4,4
+'bigint_col91','BIGINT',5,-1,8,8
+'float_col91','FLOAT',5,-1,4,4
+'double_col91','DOUBLE',5,-1,8,8
+'string_col91','STRING',5,-1,1,1
+'bool_col92','BOOLEAN',2,-1,1,1
+'tinyint_col92','TINYINT',5,-1,1,1
+'smallint_col92','SMALLINT',5,-1,2,2
+'int_col92','INT',5,-1,4,4
+'bigint_col92','BIGINT',5,-1,8,8
+'float_col92','FLOAT',5,-1,4,4
+'double_col92','DOUBLE',5,-1,8,8
+'string_col92','STRING',5,-1,1,1
+'bool_col93','BOOLEAN',2,-1,1,1
+'tinyint_col93','TINYINT',5,-1,1,1
+'smallint_col93','SMALLINT',5,-1,2,2
+'int_col93','INT',5,-1,4,4
+'bigint_col93','BIGINT',5,-1,8,8
+'float_col93','FLOAT',5,-1,4,4
+'double_col93','DOUBLE',5,-1,8,8
+'string_col93','STRING',5,-1,1,1
+'bool_col94','BOOLEAN',2,-1,1,1
+'tinyint_col94','TINYINT',5,-1,1,1
+'smallint_col94','SMALLINT',5,-1,2,2
+'int_col94','INT',5,-1,4,4
+'bigint_col94','BIGINT',5,-1,8,8
+'float_col94','FLOAT',5,-1,4,4
+'double_col94','DOUBLE',5,-1,8,8
+'string_col94','STRING',5,-1,1,1
+'bool_col95','BOOLEAN',2,-1,1,1
+'tinyint_col95','TINYINT',5,-1,1,1
+'smallint_col95','SMALLINT',5,-1,2,2
+'int_col95','INT',5,-1,4,4
+'bigint_col95','BIGINT',5,-1,8,8
+'float_col95','FLOAT',5,-1,4,4
+'double_col95','DOUBLE',5,-1,8,8
+'string_col95','STRING',5,-1,1,1
+'bool_col96','BOOLEAN',2,-1,1,1
+'tinyint_col96','TINYINT',5,-1,1,1
+'smallint_col96','SMALLINT',5,-1,2,2
+'int_col96','INT',5,-1,4,4
+'bigint_col96','BIGINT',5,-1,8,8
+'float_col96','FLOAT',5,-1,4,4
+'double_col96','DOUBLE',5,-1,8,8
+'string_col96','STRING',5,-1,1,1
+'bool_col97','BOOLEAN',2,-1,1,1
+'tinyint_col97','TINYINT',5,-1,1,1
+'smallint_col97','SMALLINT',5,-1,2,2
+'int_col97','INT',5,-1,4,4
+'bigint_col97','BIGINT',5,-1,8,8
+'float_col97','FLOAT',5,-1,4,4
+'double_col97','DOUBLE',5,-1,8,8
+'string_col97','STRING',5,-1,1,1
+'bool_col98','BOOLEAN',2,-1,1,1
+'tinyint_col98','TINYINT',5,-1,1,1
+'smallint_col98','SMALLINT',5,-1,2,2
+'int_col98','INT',5,-1,4,4
+'bigint_col98','BIGINT',5,-1,8,8
+'float_col98','FLOAT',5,-1,4,4
+'double_col98','DOUBLE',5,-1,8,8
+'string_col98','STRING',5,-1,1,1
+'bool_col99','BOOLEAN',2,-1,1,1
+'tinyint_col99','TINYINT',5,-1,1,1
+'smallint_col99','SMALLINT',5,-1,2,2
+'int_col99','INT',5,-1,4,4
+'bigint_col99','BIGINT',5,-1,8,8
+'float_col99','FLOAT',5,-1,4,4
+'double_col99','DOUBLE',5,-1,8,8
+'string_col99','STRING',5,-1,1,1
+'bool_col100','BOOLEAN',2,-1,1,1
+'tinyint_col100','TINYINT',5,-1,1,1
+'smallint_col100','SMALLINT',5,-1,2,2
+'int_col100','INT',5,-1,4,4
+'bigint_col100','BIGINT',5,-1,8,8
+'float_col100','FLOAT',5,-1,4,4
+'double_col100','DOUBLE',5,-1,8,8
+'string_col100','STRING',5,-1,1,1
+'bool_col101','BOOLEAN',2,-1,1,1
+'tinyint_col101','TINYINT',5,-1,1,1
+'smallint_col101','SMALLINT',5,-1,2,2
+'int_col101','INT',5,-1,4,4
+'bigint_col101','BIGINT',5,-1,8,8
+'float_col101','FLOAT',5,-1,4,4
+'double_col101','DOUBLE',5,-1,8,8
+'string_col101','STRING',5,-1,1,1
+'bool_col102','BOOLEAN',2,-1,1,1
+'tinyint_col102','TINYINT',5,-1,1,1
+'smallint_col102','SMALLINT',5,-1,2,2
+'int_col102','INT',5,-1,4,4
+'bigint_col102','BIGINT',5,-1,8,8
+'float_col102','FLOAT',5,-1,4,4
+'double_col102','DOUBLE',5,-1,8,8
+'string_col102','STRING',5,-1,1,1
+'bool_col103','BOOLEAN',2,-1,1,1
+'tinyint_col103','TINYINT',5,-1,1,1
+'smallint_col103','SMALLINT',5,-1,2,2
+'int_col103','INT',5,-1,4,4
+'bigint_col103','BIGINT',5,-1,8,8
+'float_col103','FLOAT',5,-1,4,4
+'double_col103','DOUBLE',5,-1,8,8
+'string_col103','STRING',5,-1,1,1
+'bool_col104','BOOLEAN',2,-1,1,1
+'tinyint_col104','TINYINT',5,-1,1,1
+'smallint_col104','SMALLINT',5,-1,2,2
+'int_col104','INT',5,-1,4,4
+'bigint_col104','BIGINT',5,-1,8,8
+'float_col104','FLOAT',5,-1,4,4
+'double_col104','DOUBLE',5,-1,8,8
+'string_col104','STRING',5,-1,1,1
+'bool_col105','BOOLEAN',2,-1,1,1
+'tinyint_col105','TINYINT',5,-1,1,1
+'smallint_col105','SMALLINT',5,-1,2,2
+'int_col105','INT',5,-1,4,4
+'bigint_col105','BIGINT',5,-1,8,8
+'float_col105','FLOAT',5,-1,4,4
+'double_col105','DOUBLE',5,-1,8,8
+'string_col105','STRING',5,-1,1,1
+'bool_col106','BOOLEAN',2,-1,1,1
+'tinyint_col106','TINYINT',5,-1,1,1
+'smallint_col106','SMALLINT',5,-1,2,2
+'int_col106','INT',5,-1,4,4
+'bigint_col106','BIGINT',5,-1,8,8
+'float_col106','FLOAT',5,-1,4,4
+'double_col106','DOUBLE',5,-1,8,8
+'string_col106','STRING',5,-1,1,1
+'bool_col107','BOOLEAN',2,-1,1,1
+'tinyint_col107','TINYINT',5,-1,1,1
+'smallint_col107','SMALLINT',5,-1,2,2
+'int_col107','INT',5,-1,4,4
+'bigint_col107','BIGINT',5,-1,8,8
+'float_col107','FLOAT',5,-1,4,4
+'double_col107','DOUBLE',5,-1,8,8
+'string_col107','STRING',5,-1,1,1
+'bool_col108','BOOLEAN',2,-1,1,1
+'tinyint_col108','TINYINT',5,-1,1,1
+'smallint_col108','SMALLINT',5,-1,2,2
+'int_col108','INT',5,-1,4,4
+'bigint_col108','BIGINT',5,-1,8,8
+'float_col108','FLOAT',5,-1,4,4
+'double_col108','DOUBLE',5,-1,8,8
+'string_col108','STRING',5,-1,1,1
+'bool_col109','BOOLEAN',2,-1,1,1
+'tinyint_col109','TINYINT',5,-1,1,1
+'smallint_col109','SMALLINT',5,-1,2,2
+'int_col109','INT',5,-1,4,4
+'bigint_col109','BIGINT',5,-1,8,8
+'float_col109','FLOAT',5,-1,4,4
+'double_col109','DOUBLE',5,-1,8,8
+'string_col109','STRING',5,-1,1,1
+'bool_col110','BOOLEAN',2,-1,1,1
+'tinyint_col110','TINYINT',5,-1,1,1
+'smallint_col110','SMALLINT',5,-1,2,2
+'int_col110','INT',5,-1,4,4
+'bigint_col110','BIGINT',5,-1,8,8
+'float_col110','FLOAT',5,-1,4,4
+'double_col110','DOUBLE',5,-1,8,8
+'string_col110','STRING',5,-1,1,1
+'bool_col111','BOOLEAN',2,-1,1,1
+'tinyint_col111','TINYINT',5,-1,1,1
+'smallint_col111','SMALLINT',5,-1,2,2
+'int_col111','INT',5,-1,4,4
+'bigint_col111','BIGINT',5,-1,8,8
+'float_col111','FLOAT',5,-1,4,4
+'double_col111','DOUBLE',5,-1,8,8
+'string_col111','STRING',5,-1,1,1
+'bool_col112','BOOLEAN',2,-1,1,1
+'tinyint_col112','TINYINT',5,-1,1,1
+'smallint_col112','SMALLINT',5,-1,2,2
+'int_col112','INT',5,-1,4,4
+'bigint_col112','BIGINT',5,-1,8,8
+'float_col112','FLOAT',5,-1,4,4
+'double_col112','DOUBLE',5,-1,8,8
+'string_col112','STRING',5,-1,1,1
+'bool_col113','BOOLEAN',2,-1,1,1
+'tinyint_col113','TINYINT',5,-1,1,1
+'smallint_col113','SMALLINT',5,-1,2,2
+'int_col113','INT',5,-1,4,4
+'bigint_col113','BIGINT',5,-1,8,8
+'float_col113','FLOAT',5,-1,4,4
+'double_col113','DOUBLE',5,-1,8,8
+'string_col113','STRING',5,-1,1,1
+'bool_col114','BOOLEAN',2,-1,1,1
+'tinyint_col114','TINYINT',5,-1,1,1
+'smallint_col114','SMALLINT',5,-1,2,2
+'int_col114','INT',5,-1,4,4
+'bigint_col114','BIGINT',5,-1,8,8
+'float_col114','FLOAT',5,-1,4,4
+'double_col114','DOUBLE',5,-1,8,8
+'string_col114','STRING',5,-1,1,1
+'bool_col115','BOOLEAN',2,-1,1,1
+'tinyint_col115','TINYINT',5,-1,1,1
+'smallint_col115','SMALLINT',5,-1,2,2
+'int_col115','INT',5,-1,4,4
+'bigint_col115','BIGINT',5,-1,8,8
+'float_col115','FLOAT',5,-1,4,4
+'double_col115','DOUBLE',5,-1,8,8
+'string_col115','STRING',5,-1,1,1
+'bool_col116','BOOLEAN',2,-1,1,1
+'tinyint_col116','TINYINT',5,-1,1,1
+'smallint_col116','SMALLINT',5,-1,2,2
+'int_col116','INT',5,-1,4,4
+'bigint_col116','BIGINT',5,-1,8,8
+'float_col116','FLOAT',5,-1,4,4
+'double_col116','DOUBLE',5,-1,8,8
+'string_col116','STRING',5,-1,1,1
+'bool_col117','BOOLEAN',2,-1,1,1
+'tinyint_col117','TINYINT',5,-1,1,1
+'smallint_col117','SMALLINT',5,-1,2,2
+'int_col117','INT',5,-1,4,4
+'bigint_col117','BIGINT',5,-1,8,8
+'float_col117','FLOAT',5,-1,4,4
+'double_col117','DOUBLE',5,-1,8,8
+'string_col117','STRING',5,-1,1,1
+'bool_col118','BOOLEAN',2,-1,1,1
+'tinyint_col118','TINYINT',5,-1,1,1
+'smallint_col118','SMALLINT',5,-1,2,2
+'int_col118','INT',5,-1,4,4
+'bigint_col118','BIGINT',5,-1,8,8
+'float_col118','FLOAT',5,-1,4,4
+'double_col118','DOUBLE',5,-1,8,8
+'string_col118','STRING',5,-1,1,1
+'bool_col119','BOOLEAN',2,-1,1,1
+'tinyint_col119','TINYINT',5,-1,1,1
+'smallint_col119','SMALLINT',5,-1,2,2
+'int_col119','INT',5,-1,4,4
+'bigint_col119','BIGINT',5,-1,8,8
+'float_col119','FLOAT',5,-1,4,4
+'double_col119','DOUBLE',5,-1,8,8
+'string_col119','STRING',5,-1,1,1
+'bool_col120','BOOLEAN',2,-1,1,1
+'tinyint_col120','TINYINT',5,-1,1,1
+'smallint_col120','SMALLINT',5,-1,2,2
+'int_col120','INT',5,-1,4,4
+'bigint_col120','BIGINT',5,-1,8,8
+'float_col120','FLOAT',5,-1,4,4
+'double_col120','DOUBLE',5,-1,8,8
+'string_col120','STRING',5,-1,1,1
+'bool_col121','BOOLEAN',2,-1,1,1
+'tinyint_col121','TINYINT',5,-1,1,1
+'smallint_col121','SMALLINT',5,-1,2,2
+'int_col121','INT',5,-1,4,4
+'bigint_col121','BIGINT',5,-1,8,8
+'float_col121','FLOAT',5,-1,4,4
+'double_col121','DOUBLE',5,-1,8,8
+'string_col121','STRING',5,-1,1,1
+'bool_col122','BOOLEAN',2,-1,1,1
+'tinyint_col122','TINYINT',5,-1,1,1
+'smallint_col122','SMALLINT',5,-1,2,2
+'int_col122','INT',5,-1,4,4
+'bigint_col122','BIGINT',5,-1,8,8
+'float_col122','FLOAT',5,-1,4,4
+'double_col122','DOUBLE',5,-1,8,8
+'string_col122','STRING',5,-1,1,1
+'bool_col123','BOOLEAN',2,-1,1,1
+'tinyint_col123','TINYINT',5,-1,1,1
+'smallint_col123','SMALLINT',5,-1,2,2
+'int_col123','INT',5,-1,4,4
+'bigint_col123','BIGINT',5,-1,8,8
+'float_col123','FLOAT',5,-1,4,4
+'double_col123','DOUBLE',5,-1,8,8
+'string_col123','STRING',5,-1,1,1
+'bool_col124','BOOLEAN',2,-1,1,1
+'tinyint_col124','TINYINT',5,-1,1,1
+'smallint_col124','SMALLINT',5,-1,2,2
+'int_col124','INT',5,-1,4,4
+'bigint_col124','BIGINT',5,-1,8,8
+'float_col124','FLOAT',5,-1,4,4
+'double_col124','DOUBLE',5,-1,8,8
+'string_col124','STRING',5,-1,1,1
+'bool_col125','BOOLEAN',2,-1,1,1
+'tinyint_col125','TINYINT',5,-1,1,1
+'smallint_col125','SMALLINT',5,-1,2,2
+'int_col125','INT',5,-1,4,4
+'bigint_col125','BIGINT',5,-1,8,8
+'float_col125','FLOAT',5,-1,4,4
+'double_col125','DOUBLE',5,-1,8,8
+'string_col125','STRING',5,-1,1,1
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, INT, DOUBLE
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4db330e6/tests/query_test/test_aggregation.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_aggregation.py b/tests/query_test/test_aggregation.py
index c44c784..38900e6 100644
--- a/tests/query_test/test_aggregation.py
+++ b/tests/query_test/test_aggregation.py
@@ -19,13 +19,19 @@
#
import pytest
+from testdata.common import widetable
from tests.common.environ import USING_OLD_AGGS_JOINS
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfOldAggsJoins
from tests.common.test_dimensions import (
create_exec_option_dimension,
create_uncompressed_text_dimension)
-from tests.common.test_result_verifier import assert_codegen_enabled
+from tests.common.test_result_verifier import (
+ assert_codegen_enabled,
+ parse_column_types,
+ parse_column_labels,
+ QueryTestResult,
+ parse_result_rows)
from tests.common.test_vector import TestDimension
# Test dimensions for TestAggregation.
@@ -128,8 +134,9 @@ class TestAggregation(ImpalaTestSuite):
self.verify_agg_result(agg_func, data_type, False, result.data[0]);
if check_codegen_enabled:
- # Verify codegen was enabled for both stages of the aggregation.
- assert_codegen_enabled(result.runtime_profile, [1, 3])
+ # Verify codegen was enabled for the preaggregation.
+ # It is deliberately disabled for the merge aggregation.
+ assert_codegen_enabled(result.runtime_profile, [1])
query = 'select %s(DISTINCT(%s_col)) from alltypesagg where day is not null' % (
agg_func, data_type)
@@ -207,7 +214,7 @@ class TestAggregationQueries(ImpalaTestSuite):
Required to run directly in python because the order in which results will be
merged at the final, single-node aggregation step is non-deterministic (if the
first phase is running on multiple nodes). Need to pull the result apart and
- compare the actual items)"""
+ compare the actual items)"""
exec_option = vector.get_value('exec_option')
disable_codegen = exec_option['disable_codegen']
table_format = vector.get_value('table_format')
@@ -263,6 +270,42 @@ class TestAggregationQueries(ImpalaTestSuite):
# Verify codegen was enabled for all four stages of the aggregation.
assert_codegen_enabled(result.runtime_profile, [1, 2, 4, 6])
+class TestWideAggregationQueries(ImpalaTestSuite):
+ """Test that aggregations with many grouping columns work"""
+ @classmethod
+ def get_workload(self):
+ return 'functional-query'
+
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestWideAggregationQueries, cls).add_test_dimensions()
+
+ cls.TestMatrix.add_dimension(
+ create_exec_option_dimension(disable_codegen_options=[False, True]))
+
+ # File format doesn't matter for this test.
+ cls.TestMatrix.add_constraint(
+ lambda v: v.get_value('table_format').file_format == 'parquet')
+
+ def test_many_grouping_columns(self, vector):
+ """Test that an aggregate with many grouping columns works"""
+ table_format = vector.get_value('table_format')
+ exec_option = vector.get_value('exec_option')
+ query = "select distinct * from widetable_1000_cols"
+
+ # Ensure codegen is enabled.
+ result = self.execute_query(query, exec_option, table_format=table_format)
+
+ # All rows should be distinct.
+ expected_result = widetable.get_data(1000, 10, quote_strings=True)
+
+ types = parse_column_types(result.schema)
+ labels = parse_column_labels(result.schema)
+ expected = QueryTestResult(expected_result, types, labels, order_matters=False)
+ actual = QueryTestResult(parse_result_rows(result), types, labels,
+ order_matters=False)
+ assert expected == actual
+
class TestTPCHAggregationQueries(ImpalaTestSuite):
# Uses the TPC-H dataset in order to have larger aggregations.
[2/6] incubator-impala git commit: IMPALA-3809: Show Kudu-specific
column metadata in DESCRIBE.
Posted by kw...@apache.org.
IMPALA-3809: Show Kudu-specific column metadata in DESCRIBE.
TODO:
- Corresponding changes to DESCRIBE EXTENDED/FORMATTED.
Testing:
A private core/hdfs run passed.
Change-Id: I83c91b540bc6d27cb4f21535fe12f3f8658c233e
Reviewed-on: http://gerrit.cloudera.org:8080/5125
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/8f2bb2f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/8f2bb2f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/8f2bb2f7
Branch: refs/heads/master
Commit: 8f2bb2f72fc94307034881f500e23f335ec08eba
Parents: 4821ea6
Author: Alex Behm <al...@cloudera.com>
Authored: Wed Nov 16 16:22:16 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 22 23:06:05 2016 +0000
----------------------------------------------------------------------
common/thrift/Frontend.thrift | 12 ++--
.../impala/analysis/DescribeTableStmt.java | 50 ++++++++------
.../impala/service/DescribeResultFactory.java | 71 ++++++++++++++++----
.../org/apache/impala/service/Frontend.java | 46 ++++++-------
.../org/apache/impala/service/JniFrontend.java | 18 +++--
.../apache/impala/analysis/AnalyzeDDLTest.java | 28 ++++----
.../queries/QueryTest/kudu_describe.test | 47 +++++++++++++
.../queries/QueryTest/kudu_stats.test | 10 +--
tests/query_test/test_kudu.py | 23 +++++--
9 files changed, 213 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8f2bb2f7/common/thrift/Frontend.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift
index 6fdf972..135fa7b 100644
--- a/common/thrift/Frontend.thrift
+++ b/common/thrift/Frontend.thrift
@@ -147,14 +147,14 @@ struct TDescribeDbParams {
// given TDescribeOutputStyle.
// NOTE: This struct should only be used for intra-process communication.
struct TDescribeTableParams {
- 1: required string db
- 2: required string table_name
-
// Controls the output style for this describe command.
- 3: required TDescribeOutputStyle output_style
+ 1: required TDescribeOutputStyle output_style
+
+ // Set when describing a table.
+ 2: optional CatalogObjects.TTableName table_name
- // Struct type with fields to display for the MINIMAL output style.
- 4: optional Types.TColumnType result_struct
+ // Set when describing a path to a nested collection.
+ 3: optional Types.TColumnType result_struct
}
// Results of a call to describeDb() and describeTable()
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8f2bb2f7/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java b/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java
index b947e8e..6977f3b 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java
@@ -20,29 +20,30 @@ package org.apache.impala.analysis;
import java.util.ArrayList;
import org.apache.commons.lang3.StringUtils;
-
-import parquet.Strings;
-
import org.apache.impala.analysis.Path.PathType;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.authorization.PrivilegeRequestBuilder;
import org.apache.impala.catalog.StructType;
+import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.TableLoadingException;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TDescribeOutputStyle;
import org.apache.impala.thrift.TDescribeTableParams;
+
+import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
/**
- * Representation of a DESCRIBE table statement which returns metadata on
- * a specified table:
+ * Representation of a DESCRIBE statement which returns metadata on a specified
+ * table or path:
* Syntax: DESCRIBE <path>
* DESCRIBE FORMATTED|EXTENDED <table>
*
* If FORMATTED|EXTENDED is not specified and the path refers to a table, the statement
- * only returns info on the given table's column definition (column name, data type, and
- * comment). If the path refers to a complex typed field within a column, the statement
- * returns the field names, types, and comments.
+ * only returns info on the given table's column definitions (column name, data type,
+ * comment, and table-type-specific info like nullability, etc.). If the path refers to
+ * a complex typed field within a column, the statement returns the field names, types,
+ * and comments.
* If FORMATTED|EXTENDED is specified, extended metadata on the table is returned
* (in addition to the column definitions). This metadata includes info about the table
* properties, SerDe properties, StorageDescriptor properties, and more.
@@ -51,15 +52,16 @@ public class DescribeTableStmt extends StatementBase {
private final TDescribeOutputStyle outputStyle_;
/// "."-separated path from the describe statement.
- private ArrayList<String> rawPath_;
+ private final ArrayList<String> rawPath_;
/// The resolved path to describe, set after analysis.
private Path path_;
/// The fully qualified name of the root table, set after analysis.
- private TableName tableName_;
+ private Table table_;
/// Struct type with the fields to display for the described path.
+ /// Only set when describing a path to a nested collection.
private StructType resultStruct_;
public DescribeTableStmt(ArrayList<String> rawPath, TDescribeOutputStyle outputStyle) {
@@ -68,7 +70,6 @@ public class DescribeTableStmt extends StatementBase {
rawPath_ = rawPath;
outputStyle_ = outputStyle;
path_ = null;
- tableName_ = null;
resultStruct_ = null;
}
@@ -81,10 +82,9 @@ public class DescribeTableStmt extends StatementBase {
return sb.toString() + StringUtils.join(rawPath_, ".");
}
- public TableName getTableName() { return tableName_; }
+ public Table getTable() { return table_; }
public TDescribeOutputStyle getOutputStyle() { return outputStyle_; }
-
/**
* Get the privilege requirement, which depends on the output style.
*/
@@ -122,31 +122,37 @@ public class DescribeTableStmt extends StatementBase {
throw new AnalysisException(tle.getMessage(), tle);
}
- tableName_ = analyzer.getFqTableName(path_.getRootTable().getTableName());
- analyzer.getTable(tableName_, getPrivilegeRequirement());
+ table_ = path_.getRootTable();
+ // Register authorization and audit events.
+ analyzer.getTable(table_.getTableName(), getPrivilegeRequirement());
- if (path_.destTable() != null) {
- resultStruct_ = path_.getRootTable().getHiveColumnsAsStruct();
- } else if (path_.destType().isComplexType()) {
+ // Describing a table.
+ if (path_.destTable() != null) return;
+
+ if (path_.destType().isComplexType()) {
if (outputStyle_ == TDescribeOutputStyle.FORMATTED ||
outputStyle_ == TDescribeOutputStyle.EXTENDED) {
throw new AnalysisException("DESCRIBE FORMATTED|EXTENDED must refer to a table");
}
+ // Describing a nested collection.
Preconditions.checkState(outputStyle_ == TDescribeOutputStyle.MINIMAL);
resultStruct_ = Path.getTypeAsStruct(path_.destType());
} else {
throw new AnalysisException("Cannot describe path '" +
- Strings.join(rawPath_, ".") + "' targeting scalar type: " +
+ Joiner.on('.').join(rawPath_) + "' targeting scalar type: " +
path_.destType().toSql());
}
}
public TDescribeTableParams toThrift() {
TDescribeTableParams params = new TDescribeTableParams();
- params.setTable_name(getTableName().getTbl());
- params.setDb(getTableName().getDb());
params.setOutput_style(outputStyle_);
- params.setResult_struct(resultStruct_.toThrift());
+ if (resultStruct_ != null) {
+ params.setResult_struct(resultStruct_.toThrift());
+ } else {
+ Preconditions.checkNotNull(table_);
+ params.setTable_name(table_.getTableName().toThrift());
+ }
return params;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8f2bb2f7/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java b/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
index eb917a6..05e05f9 100644
--- a/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
+++ b/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
@@ -25,9 +25,10 @@ import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatUtils;
-
import org.apache.impala.catalog.Column;
import org.apache.impala.catalog.Db;
+import org.apache.impala.catalog.KuduColumn;
+import org.apache.impala.catalog.KuduTable;
import org.apache.impala.catalog.StructField;
import org.apache.impala.catalog.StructType;
import org.apache.impala.catalog.Table;
@@ -35,9 +36,12 @@ import org.apache.impala.thrift.TColumnValue;
import org.apache.impala.thrift.TDescribeOutputStyle;
import org.apache.impala.thrift.TDescribeResult;
import org.apache.impala.thrift.TResultRow;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import com.google.common.collect.Lists;
-/*
+/**
* Builds results for DESCRIBE DATABASE statements by constructing and
* populating a TDescribeResult object.
*/
@@ -173,15 +177,15 @@ public class DescribeResultFactory {
return descResult;
}
- /*
+ /**
* Builds a TDescribeResult that contains the result of a DESCRIBE FORMATTED|EXTENDED
* <table> command. For the formatted describe output the goal is to be exactly the
* same as what Hive (via HiveServer2) outputs, for compatibility reasons. To do this,
* Hive's MetadataFormatUtils class is used to build the results.
*/
public static TDescribeResult buildDescribeFormattedResult(Table table) {
- TDescribeResult descResult = new TDescribeResult();
- descResult.results = Lists.newArrayList();
+ TDescribeResult result = new TDescribeResult();
+ result.results = Lists.newArrayList();
org.apache.hadoop.hive.metastore.api.Table msTable =
table.getMetaStoreTable().deepCopy();
@@ -218,29 +222,72 @@ public class DescribeResultFactory {
}
resultRow.addToColVals(colVal);
}
- descResult.results.add(resultRow);
+ result.results.add(resultRow);
}
- return descResult;
+ return result;
}
- /*
- * Builds a TDescribeResult that contains the result of a DESCRIBE <path> command:
- * the names and types of fields of the table or complex type referred to by the path.
+ /**
+ * Builds a TDescribeResult for a nested collection whose fields are represented
+ * by the given StructType.
*/
public static TDescribeResult buildDescribeMinimalResult(StructType type) {
TDescribeResult descResult = new TDescribeResult();
descResult.results = Lists.newArrayList();
-
for (StructField field: type.getFields()) {
TColumnValue colNameCol = new TColumnValue();
colNameCol.setString_val(field.getName());
TColumnValue dataTypeCol = new TColumnValue();
dataTypeCol.setString_val(field.getType().prettyPrint().toLowerCase());
TColumnValue commentCol = new TColumnValue();
- commentCol.setString_val(field.getComment() != null ? field.getComment() : "");
+ commentCol.setString_val(Strings.nullToEmpty(field.getComment()));
descResult.results.add(
new TResultRow(Lists.newArrayList(colNameCol, dataTypeCol, commentCol)));
}
return descResult;
}
+
+ /**
+ * Builds a TDescribeResult for a table.
+ */
+ public static TDescribeResult buildDescribeMinimalResult(Table table) {
+ if (!(table instanceof KuduTable)) {
+ return buildDescribeMinimalResult(table.getHiveColumnsAsStruct());
+ }
+
+ TDescribeResult descResult = new TDescribeResult();
+ descResult.results = Lists.newArrayList();
+ for (Column c: table.getColumnsInHiveOrder()) {
+ Preconditions.checkState(c instanceof KuduColumn);
+ KuduColumn kuduColumn = (KuduColumn) c;
+ // General describe info.
+ TColumnValue colNameCol = new TColumnValue();
+ colNameCol.setString_val(kuduColumn.getName());
+ TColumnValue dataTypeCol = new TColumnValue();
+ dataTypeCol.setString_val(kuduColumn.getType().prettyPrint().toLowerCase());
+ TColumnValue commentCol = new TColumnValue();
+ commentCol.setString_val(Strings.nullToEmpty(kuduColumn.getComment()));
+ // Kudu-specific describe info.
+ TColumnValue pkCol = new TColumnValue();
+ pkCol.setString_val(Boolean.toString(kuduColumn.isKey()));
+ TColumnValue nullableCol = new TColumnValue();
+ nullableCol.setString_val(Boolean.toString(kuduColumn.isNullable()));
+ TColumnValue defaultValCol = new TColumnValue();
+ if (kuduColumn.hasDefaultValue()) {
+ defaultValCol.setString_val(kuduColumn.getDefaultValue().getStringValue());
+ } else {
+ defaultValCol.setString_val("");
+ }
+ TColumnValue encodingCol = new TColumnValue();
+ encodingCol.setString_val(kuduColumn.getEncoding().toString());
+ TColumnValue compressionCol = new TColumnValue();
+ compressionCol.setString_val(kuduColumn.getCompression().toString());
+ TColumnValue blockSizeCol = new TColumnValue();
+ blockSizeCol.setString_val(Integer.toString(kuduColumn.getBlockSize()));
+ descResult.results.add(new TResultRow(
+ Lists.newArrayList(colNameCol, dataTypeCol, commentCol, pkCol, nullableCol,
+ defaultValCol, encodingCol, compressionCol, blockSizeCol)));
+ }
+ return descResult;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8f2bb2f7/fe/src/main/java/org/apache/impala/service/Frontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index 8657182..5e0307c 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -20,13 +20,11 @@ package org.apache.impala.service;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
@@ -35,22 +33,18 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
-import org.apache.impala.catalog.KuduTable;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hive.service.cli.thrift.TGetColumnsReq;
import org.apache.hive.service.cli.thrift.TGetFunctionsReq;
import org.apache.hive.service.cli.thrift.TGetSchemasReq;
import org.apache.hive.service.cli.thrift.TGetTablesReq;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.impala.analysis.AnalysisContext;
-import org.apache.impala.analysis.Analyzer;
import org.apache.impala.analysis.CreateDataSrcStmt;
import org.apache.impala.analysis.CreateDropRoleStmt;
import org.apache.impala.analysis.CreateUdaStmt;
import org.apache.impala.analysis.CreateUdfStmt;
+import org.apache.impala.analysis.DescribeTableStmt;
import org.apache.impala.analysis.DescriptorTable;
import org.apache.impala.analysis.DropDataSrcStmt;
import org.apache.impala.analysis.DropFunctionStmt;
@@ -66,7 +60,6 @@ import org.apache.impala.analysis.ShowGrantRoleStmt;
import org.apache.impala.analysis.ShowRolesStmt;
import org.apache.impala.analysis.TableName;
import org.apache.impala.analysis.TruncateStmt;
-import org.apache.impala.analysis.TupleDescriptor;
import org.apache.impala.authorization.AuthorizationChecker;
import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.authorization.ImpalaInternalAdminUser;
@@ -85,7 +78,7 @@ import org.apache.impala.catalog.Function;
import org.apache.impala.catalog.HBaseTable;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.ImpaladCatalog;
-import org.apache.impala.catalog.StructType;
+import org.apache.impala.catalog.KuduTable;
import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
@@ -93,7 +86,6 @@ import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.NotImplementedException;
-import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.planner.PlanFragment;
import org.apache.impala.planner.Planner;
import org.apache.impala.planner.ScanNode;
@@ -101,7 +93,6 @@ import org.apache.impala.thrift.TCatalogOpRequest;
import org.apache.impala.thrift.TCatalogOpType;
import org.apache.impala.thrift.TCatalogServiceRequestHeader;
import org.apache.impala.thrift.TColumn;
-import org.apache.impala.thrift.TColumnType;
import org.apache.impala.thrift.TColumnValue;
import org.apache.impala.thrift.TCreateDropRoleParams;
import org.apache.impala.thrift.TDdlExecRequest;
@@ -110,7 +101,6 @@ import org.apache.impala.thrift.TDescribeOutputStyle;
import org.apache.impala.thrift.TDescribeResult;
import org.apache.impala.thrift.TErrorCode;
import org.apache.impala.thrift.TExecRequest;
-import org.apache.impala.thrift.TExplainLevel;
import org.apache.impala.thrift.TExplainResult;
import org.apache.impala.thrift.TFinalizeParams;
import org.apache.impala.thrift.TFunctionCategory;
@@ -122,7 +112,6 @@ import org.apache.impala.thrift.TLoadDataResp;
import org.apache.impala.thrift.TMetadataOpRequest;
import org.apache.impala.thrift.TPlanExecInfo;
import org.apache.impala.thrift.TPlanFragment;
-import org.apache.impala.thrift.TPlanFragmentTree;
import org.apache.impala.thrift.TQueryCtx;
import org.apache.impala.thrift.TQueryExecRequest;
import org.apache.impala.thrift.TResetMetadataRequest;
@@ -141,11 +130,13 @@ import org.apache.impala.util.MembershipSnapshot;
import org.apache.impala.util.PatternMatcher;
import org.apache.impala.util.TResultRowBuilder;
import org.apache.impala.util.TSessionStateUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicates;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
/**
@@ -319,11 +310,22 @@ public class Frontend {
new TColumn("comment", Type.STRING.toThrift())));
} else if (analysis.isDescribeTableStmt()) {
ddl.op_type = TCatalogOpType.DESCRIBE_TABLE;
- ddl.setDescribe_table_params(analysis.getDescribeTableStmt().toThrift());
- metadata.setColumns(Arrays.asList(
+ DescribeTableStmt descStmt = analysis.getDescribeTableStmt();
+ ddl.setDescribe_table_params(descStmt.toThrift());
+ List<TColumn> columns = Lists.newArrayList(
new TColumn("name", Type.STRING.toThrift()),
new TColumn("type", Type.STRING.toThrift()),
- new TColumn("comment", Type.STRING.toThrift())));
+ new TColumn("comment", Type.STRING.toThrift()));
+ if (descStmt.getTable() instanceof KuduTable
+ && descStmt.getOutputStyle() == TDescribeOutputStyle.MINIMAL) {
+ columns.add(new TColumn("primary_key", Type.STRING.toThrift()));
+ columns.add(new TColumn("nullable", Type.STRING.toThrift()));
+ columns.add(new TColumn("default_value", Type.STRING.toThrift()));
+ columns.add(new TColumn("encoding", Type.STRING.toThrift()));
+ columns.add(new TColumn("compression", Type.STRING.toThrift()));
+ columns.add(new TColumn("block_size", Type.STRING.toThrift()));
+ }
+ metadata.setColumns(columns);
} else if (analysis.isAlterTableStmt()) {
ddl.op_type = TCatalogOpType.DDL;
TDdlExecRequest req = new TDdlExecRequest();
@@ -773,16 +775,14 @@ public class Frontend {
* Throws an exception if the table or db is not found or if there is an error loading
* the table metadata.
*/
- public TDescribeResult describeTable(String dbName, String tableName,
- TDescribeOutputStyle outputStyle, TColumnType tResultStruct)
- throws ImpalaException {
+ public TDescribeResult describeTable(TTableName tableName,
+ TDescribeOutputStyle outputStyle) throws ImpalaException {
+ Table table = impaladCatalog_.getTable(tableName.db_name, tableName.table_name);
if (outputStyle == TDescribeOutputStyle.MINIMAL) {
- StructType resultStruct = (StructType)Type.fromThrift(tResultStruct);
- return DescribeResultFactory.buildDescribeMinimalResult(resultStruct);
+ return DescribeResultFactory.buildDescribeMinimalResult(table);
} else {
Preconditions.checkArgument(outputStyle == TDescribeOutputStyle.FORMATTED ||
outputStyle == TDescribeOutputStyle.EXTENDED);
- Table table = impaladCatalog_.getTable(dbName, tableName);
return DescribeResultFactory.buildDescribeFormattedResult(table);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8f2bb2f7/fe/src/main/java/org/apache/impala/service/JniFrontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
index 5e41af4..b343369 100644
--- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
+++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -42,15 +42,18 @@ import org.apache.impala.catalog.DataSource;
import org.apache.impala.catalog.Db;
import org.apache.impala.catalog.Function;
import org.apache.impala.catalog.Role;
+import org.apache.impala.catalog.StructType;
+import org.apache.impala.catalog.Type;
import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.JniUtil;
-import org.apache.impala.service.BackendConfig;
+import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.thrift.TBuildTestDescriptorTableParams;
import org.apache.impala.thrift.TCatalogObject;
import org.apache.impala.thrift.TDatabase;
import org.apache.impala.thrift.TDescribeDbParams;
+import org.apache.impala.thrift.TDescribeOutputStyle;
import org.apache.impala.thrift.TDescribeResult;
import org.apache.impala.thrift.TDescribeTableParams;
import org.apache.impala.thrift.TDescriptorTable;
@@ -82,7 +85,6 @@ import org.apache.impala.thrift.TTableName;
import org.apache.impala.thrift.TUniqueId;
import org.apache.impala.thrift.TUpdateCatalogCacheRequest;
import org.apache.impala.thrift.TUpdateMembershipRequest;
-import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.util.GlogAppender;
import org.apache.impala.util.PatternMatcher;
import org.apache.impala.util.TSessionStateUtil;
@@ -447,9 +449,15 @@ public class JniFrontend {
TDescribeTableParams params = new TDescribeTableParams();
JniUtil.deserializeThrift(protocolFactory_, params, thriftDescribeTableParams);
- TDescribeResult result = frontend_.describeTable(
- params.getDb(), params.getTable_name(), params.getOutput_style(),
- params.getResult_struct());
+ Preconditions.checkState(params.isSetTable_name() ^ params.isSetResult_struct());
+ TDescribeResult result = null;
+ if (params.isSetTable_name()) {
+ result = frontend_.describeTable(params.getTable_name(), params.output_style);
+ } else {
+ Preconditions.checkState(params.output_style == TDescribeOutputStyle.MINIMAL);
+ StructType structType = (StructType)Type.fromThrift(params.result_struct);
+ result = DescribeResultFactory.buildDescribeMinimalResult(structType);
+ }
TSerializer serializer = new TSerializer(protocolFactory_);
try {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8f2bb2f7/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
index ed900bf..dbfdd60 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -39,17 +39,17 @@ import org.apache.impala.catalog.DataSourceTable;
import org.apache.impala.catalog.KuduTable;
import org.apache.impala.catalog.PrimitiveType;
import org.apache.impala.catalog.ScalarType;
-import org.apache.impala.catalog.StructField;
-import org.apache.impala.catalog.StructType;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.FrontendTestBase;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.testutil.TestUtils;
+import org.apache.impala.thrift.TDescribeTableParams;
import org.apache.impala.util.MetaStoreUtil;
import org.apache.kudu.ColumnSchema.CompressionAlgorithm;
import org.apache.kudu.ColumnSchema.Encoding;
+import org.junit.Assert;
import org.junit.Test;
import com.google.common.base.Joiner;
@@ -57,8 +57,6 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
-import junit.framework.Assert;
-
public class AnalyzeDDLTest extends FrontendTestBase {
@Test
@@ -3090,14 +3088,11 @@ public class AnalyzeDDLTest extends FrontendTestBase {
// Single element path can only be resolved as <table>.
DescribeTableStmt describe = (DescribeTableStmt)AnalyzesOk("describe ambig",
createAnalyzer("ambig"));
- Assert.assertEquals("ambig", describe.toThrift().db);
- Assert.assertEquals("ambig", describe.toThrift().table_name, "ambig");
- StructType colStructType = new StructType(Lists.newArrayList(
- new StructField("ambig", new ArrayType(Type.INT))));
- StructType tableStructType = new StructType(Lists.newArrayList(
- new StructField("ambig", colStructType)));
- Assert.assertEquals(tableStructType.toSql(),
- Type.fromThrift(describe.toThrift().result_struct).toSql());
+ TDescribeTableParams tdesc = (TDescribeTableParams) describe.toThrift();
+ Assert.assertTrue(tdesc.isSetTable_name());
+ Assert.assertEquals("ambig", tdesc.table_name.getDb_name());
+ Assert.assertEquals("ambig", tdesc.table_name.getTable_name(), "ambig");
+ Assert.assertFalse(tdesc.isSetResult_struct());
// Path could be resolved as either <db>.<table> or <table>.<complex field>
AnalysisError("describe ambig.ambig", createAnalyzer("ambig"),
@@ -3106,8 +3101,13 @@ public class AnalyzeDDLTest extends FrontendTestBase {
AnalysisError("describe ambig.ambig.ambig", createAnalyzer("ambig"),
"Path is ambiguous: 'ambig.ambig.ambig'");
// 4 element path can only be resolved to nested array.
- AnalyzesOk("describe ambig.ambig.ambig.ambig", createAnalyzer("ambig"));
-
+ describe = (DescribeTableStmt) AnalyzesOk(
+ "describe ambig.ambig.ambig.ambig", createAnalyzer("ambig"));
+ tdesc = (TDescribeTableParams) describe.toThrift();
+ Type expectedType =
+ org.apache.impala.analysis.Path.getTypeAsStruct(new ArrayType(Type.INT));
+ Assert.assertTrue(tdesc.isSetResult_struct());
+ Assert.assertEquals(expectedType, Type.fromThrift(tdesc.getResult_struct()));
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8f2bb2f7/testdata/workloads/functional-query/queries/QueryTest/kudu_describe.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_describe.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_describe.test
new file mode 100644
index 0000000..280c87d
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_describe.test
@@ -0,0 +1,47 @@
+====
+---- QUERY
+describe functional_kudu.alltypes
+---- LABELS
+NAME,TYPE,COMMENT,PRIMARY_KEY,NULLABLE,DEFAULT_VALUE,ENCODING,COMPRESSION,BLOCK_SIZE
+---- RESULTS
+'bigint_col','bigint','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'bool_col','boolean','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'date_string_col','string','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'double_col','double','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'float_col','float','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'id','int','','true','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'int_col','int','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'month','int','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'smallint_col','smallint','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'string_col','string','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'timestamp_col','string','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'tinyint_col','tinyint','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'year','int','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+---- TYPES
+STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING
+====
+---- QUERY
+# Test composite primary key and column options.
+create table describe_test
+(pk1 int,
+ pk2 int,
+ pk3 string,
+ c1 string null default 'abc' comment 'testing',
+ c2 int default 100 encoding plain_encoding compression snappy,
+ c3 int null block_size 8388608,
+ primary key (pk1, pk2, pk3))
+distribute by hash (pk1) into 3 buckets
+stored as kudu;
+describe describe_test;
+---- LABELS
+NAME,TYPE,COMMENT,PRIMARY_KEY,NULLABLE,DEFAULT_VALUE,ENCODING,COMPRESSION,BLOCK_SIZE
+---- RESULTS
+'pk1','int','','true','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'pk2','int','','true','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'pk3','string','','true','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'c1','string','','false','true','abc','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'c2','int','','false','false','100','PLAIN_ENCODING','SNAPPY','0'
+'c3','int','','false','true','','AUTO_ENCODING','DEFAULT_COMPRESSION','8388608'
+---- TYPES
+STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8f2bb2f7/testdata/workloads/functional-query/queries/QueryTest/kudu_stats.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_stats.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_stats.test
index 6914944..3ae4f69 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu_stats.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_stats.test
@@ -22,10 +22,10 @@ INT,STRING,STRING,STRING,INT
compute stats simple;
describe simple;
---- RESULTS
-'id','int',''
-'name','string',''
-'valf','float',''
-'vali','bigint',''
+'id','int','','true','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'name','string','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'valf','float','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
+'vali','bigint','','false','false','','AUTO_ENCODING','DEFAULT_COMPRESSION','0'
---- TYPES
-STRING,STRING,STRING
+STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING,STRING
====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8f2bb2f7/tests/query_test/test_kudu.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py
index 996931d..4ea8770 100644
--- a/tests/query_test/test_kudu.py
+++ b/tests/query_test/test_kudu.py
@@ -64,6 +64,9 @@ class TestKuduOperations(KuduTestSuite):
def test_kudu_stats(self, vector, unique_database):
self.run_test_case('QueryTest/kudu_stats', vector, use_db=unique_database)
+ def test_kudu_describe(self, vector, unique_database):
+ self.run_test_case('QueryTest/kudu_describe', vector, use_db=unique_database)
+
def test_kudu_column_options(self, cursor, kudu_client, unique_database):
encodings = ["ENCODING PLAIN_ENCODING", ""]
compressions = ["COMPRESSION SNAPPY", ""]
@@ -120,7 +123,7 @@ class TestCreateExternalTable(KuduTestSuite):
with self.drop_impala_table_after_context(cursor, impala_table_name):
cursor.execute("DESCRIBE %s" % impala_table_name)
kudu_schema = kudu_table.schema
- for i, (col_name, col_type, _) in enumerate(cursor):
+ for i, (col_name, col_type, _, _, _, _, _, _, _) in enumerate(cursor):
kudu_col = kudu_schema[i]
assert col_name == kudu_col.name
assert col_type.upper() == \
@@ -199,7 +202,9 @@ class TestCreateExternalTable(KuduTestSuite):
impala_table_name, preferred_kudu_table.name))
with self.drop_impala_table_after_context(cursor, impala_table_name):
cursor.execute("DESCRIBE %s" % impala_table_name)
- assert cursor.fetchall() == [("a", "bigint", "")]
+ assert cursor.fetchall() == \
+ [("a", "bigint", "", "true", "false", "", "AUTO_ENCODING",
+ "DEFAULT_COMPRESSION", "0")]
def test_explicit_name_doesnt_exist(self, cursor, kudu_client):
kudu_table_name = self.random_table_name()
@@ -432,7 +437,9 @@ class TestImpalaKuduIntegration(KuduTestSuite):
cursor.execute("CREATE EXTERNAL TABLE %s STORED AS KUDU %s" % (
impala_table_name, props))
cursor.execute("DESCRIBE %s" % (impala_table_name))
- assert cursor.fetchall() == [("a", "int", "")]
+ assert cursor.fetchall() == \
+ [("a", "int", "", "true", "false", "", "AUTO_ENCODING",
+ "DEFAULT_COMPRESSION", "0")]
# Drop the underlying Kudu table and replace it with another Kudu table that has
# the same name but different schema
@@ -448,7 +455,11 @@ class TestImpalaKuduIntegration(KuduTestSuite):
# Kudu.
cursor.execute("REFRESH %s" % (impala_table_name))
cursor.execute("DESCRIBE %s" % (impala_table_name))
- assert cursor.fetchall() == [("b", "string", ""), ("c", "string", "")]
+ assert cursor.fetchall() == \
+ [("b", "string", "", "true", "false", "", "AUTO_ENCODING",
+ "DEFAULT_COMPRESSION", "0"),
+ ("c", "string", "", "false", "true", "", "AUTO_ENCODING",
+ "DEFAULT_COMPRESSION", "0")]
def test_delete_external_kudu_table(self, cursor, kudu_client):
"""Check that Impala can recover from the case where the underlying Kudu table of
@@ -462,7 +473,9 @@ class TestImpalaKuduIntegration(KuduTestSuite):
cursor.execute("CREATE EXTERNAL TABLE %s STORED AS KUDU %s" % (
impala_table_name, props))
cursor.execute("DESCRIBE %s" % (impala_table_name))
- assert cursor.fetchall() == [("a", "int", "")]
+ assert cursor.fetchall() == \
+ [("a", "int", "", "true", "false", "", "AUTO_ENCODING",
+ "DEFAULT_COMPRESSION", "0")]
# Drop the underlying Kudu table
kudu_client.delete_table(kudu_table.name)
assert not kudu_client.table_exists(kudu_table.name)