You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jr...@apache.org on 2017/02/08 21:59:45 UTC
[1/5] incubator-impala git commit: IMPALA-4880: Clarify
synchronization policy for 'done_' in KuduScanNode
Repository: incubator-impala
Updated Branches:
refs/heads/master 6a9df5409 -> 8de2884e8
IMPALA-4880: Clarify synchronization policy for 'done_' in KuduScanNode
This patch adds a comment to clarify that it's safe to optimistically
read 'done_' in some places in KuduScanNode. This is to avoid some
confusion as the code synchronizes access to the variable in certain
places but not in others.
Also, did some miscellaneous cleanup in a couple of places.
Change-Id: I72c5f1fecf4bcf737d71a1fa13e59361604485f0
Reviewed-on: http://gerrit.cloudera.org:8080/5494
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/36711759
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/36711759
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/36711759
Branch: refs/heads/master
Commit: 367117594ab7e7be0176601cf74962e875afa766
Parents: 6a9df54
Author: Sailesh Mukil <sa...@cloudera.com>
Authored: Tue Dec 13 16:14:57 2016 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Feb 7 00:07:17 2017 +0000
----------------------------------------------------------------------
be/src/exec/kudu-scan-node.cc | 22 +++++++++-------------
be/src/exec/kudu-scan-node.h | 5 ++++-
2 files changed, 13 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36711759/be/src/exec/kudu-scan-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-scan-node.cc b/be/src/exec/kudu-scan-node.cc
index 40689ee..630f5b7 100644
--- a/be/src/exec/kudu-scan-node.cc
+++ b/be/src/exec/kudu-scan-node.cc
@@ -183,12 +183,8 @@ Status KuduScanNode::GetNext(RuntimeState* state, RowBatch* row_batch, bool* eos
*eos = true;
}
- Status status;
- {
- unique_lock<mutex> l(lock_);
- status = status_;
- }
- return status;
+ unique_lock<mutex> l(lock_);
+ return status_;
}
void KuduScanNode::Close(RuntimeState* state) {
@@ -219,7 +215,7 @@ void KuduScanNode::DebugString(int indentation_level, stringstream* out) const {
const string* KuduScanNode::GetNextScanToken() {
unique_lock<mutex> lock(lock_);
- if (next_scan_token_idx_ >= scan_tokens_.size()) return NULL;
+ if (done_ || next_scan_token_idx_ >= scan_tokens_.size()) return nullptr;
const string* token = &scan_tokens_[next_scan_token_idx_++];
return token;
}
@@ -254,7 +250,7 @@ void KuduScanNode::ThreadAvailableCb(ThreadResourceMgr::ResourcePool* pool) {
Status KuduScanNode::ProcessScanToken(KuduScanner* scanner, const string& scan_token) {
RETURN_IF_ERROR(scanner->OpenNextScanToken(scan_token));
bool eos = false;
- while (!eos) {
+ while (!eos && !done_) {
gscoped_ptr<RowBatch> row_batch(new RowBatch(
row_desc(), runtime_state_->batch_size(), mem_tracker()));
RETURN_IF_ERROR(scanner->GetNext(row_batch.get(), &eos));
@@ -282,6 +278,8 @@ void KuduScanNode::RunScannerThread(const string& name, const string* initial_to
const string* scan_token = initial_token;
Status status = scanner.Open();
if (status.ok()) {
+ // Here, even though a read of 'done_' may conflict with a write to it,
+ // ProcessScanToken() will return early, as will GetNextScanToken().
while (!done_ && scan_token != NULL) {
status = ProcessScanToken(&scanner, *scan_token);
if (!status.ok()) break;
@@ -304,11 +302,9 @@ void KuduScanNode::RunScannerThread(const string& name, const string* initial_to
{
unique_lock<mutex> l(lock_);
- if (!status.ok()) {
- if (status_.ok()) {
- status_ = status;
- done_ = true;
- }
+ if (!status.ok() && status_.ok()) {
+ status_ = status;
+ done_ = true;
}
// Decrement num_active_scanners_ unless handling the case of an early exit when
// optional threads have been exceeded, in which case it already was decremented.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36711759/be/src/exec/kudu-scan-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-scan-node.h b/be/src/exec/kudu-scan-node.h
index 84e2531..3b896c8 100644
--- a/be/src/exec/kudu-scan-node.h
+++ b/be/src/exec/kudu-scan-node.h
@@ -93,7 +93,10 @@ class KuduScanNode : public ScanNode {
/// Set to true when the scan is complete (either because all scan tokens have been
/// processed, the limit was reached or some error occurred).
- /// Protected by lock_
+ /// Protected by lock_. It is safe to do optimistic reads without taking lock_ in
+ /// certain places, based on the decisions taken after that.
+ /// The tradeoff is occasionally doing some extra work versus increasing lock
+ /// contention.
volatile bool done_;
/// Thread group for all scanner worker threads
[5/5] incubator-impala git commit: IMPALA-4810: Make DECIMAL
expr-test cases table driven
Posted by jr...@apache.org.
IMPALA-4810: Make DECIMAL expr-test cases table driven
That way, we can easily run all test cases with both:
DECIMAL_V2={false, true}
Currently, the behavior is the same, but as we work on IMPALA-4810,
the expected results will differ.
Change-Id: Ieffb2573de46bba64d1b4e8caf15cc0238a84ea1
Reviewed-on: http://gerrit.cloudera.org:8080/5933
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/8de2884e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/8de2884e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/8de2884e
Branch: refs/heads/master
Commit: 8de2884e8ba635c51af4f5d6fe8d12a369db4bd6
Parents: fcc2d81
Author: Dan Hecht <dh...@cloudera.com>
Authored: Tue Feb 7 15:04:30 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Feb 8 04:38:22 2017 +0000
----------------------------------------------------------------------
be/src/exprs/expr-test.cc | 180 ++++++++++++++++----------
be/src/testutil/impalad-query-executor.h | 11 +-
2 files changed, 120 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8de2884e/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 8624a85..e6cfcdb 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -402,7 +402,8 @@ class ExprTest : public testing::Test {
// Decimals don't work with TestValue.
// TODO: figure out what operators need to be implemented to work with EXPECT_EQ
- template <typename T>
+ // TODO: verify that the result type has the same precision/scale as 'expected_type'.
+ template<typename T>
void TestDecimalValue(const string& expr, const T& expected_result,
const ColumnType& expected_type) {
const string value = GetValue(expr, expected_type);
@@ -1265,64 +1266,108 @@ TEST_F(ExprTest, ArithmeticExprs) {
TestFactorialArithmeticOp();
}
-TEST_F(ExprTest, DecimalArithmeticExprs) {
- TestDecimalValue("1.23 + cast(1 as decimal(4,3))",
- Decimal4Value(2230), ColumnType::CreateDecimalType(5,3));
- TestDecimalValue("1.23 - cast(0.23 as decimal(10,3))",
- Decimal4Value(1000), ColumnType::CreateDecimalType(11,3));
- TestDecimalValue("1.23 * cast(1 as decimal(20,3))",
- Decimal4Value(123000), ColumnType::CreateDecimalType(23,5));
- TestDecimalValue("cast(1.23 as decimal(8,2)) / cast(1 as decimal(4,3))",
- Decimal4Value(12300000), ColumnType::CreateDecimalType(16,7));
- TestDecimalValue("cast(1.23 as decimal(8,2)) % cast(1 as decimal(10,3))",
- Decimal4Value(230), ColumnType::CreateDecimalType(9,3));
- TestDecimalValue("cast(1.23 as decimal(8,2)) + cast(1 as decimal(20,3))",
- Decimal4Value(2230), ColumnType::CreateDecimalType(21, 3));
- TestDecimalValue("cast(1.23 as decimal(30,2)) - cast(1 as decimal(4,3))",
- Decimal4Value(230), ColumnType::CreateDecimalType(34,3));
- TestDecimalValue("cast(1.23 as decimal(30,2)) * cast(1 as decimal(10,3))",
- Decimal4Value(123000), ColumnType::CreateDecimalType(38,5));
- TestDecimalValue("cast(1.23 as decimal(30,2)) / cast(1 as decimal(20,3))",
- Decimal4Value(1230), ColumnType::CreateDecimalType(38, 3));
- TestDecimalValue("cast(1 as decimal(38,0)) + cast(.2 as decimal(38,1))",
- Decimal4Value(12), ColumnType::CreateDecimalType(38, 1));
- TestDecimalValue("cast(1 as decimal(38,0)) / cast(.2 as decimal(38,1))",
- Decimal4Value(50), ColumnType::CreateDecimalType(38, 1));
-
- // Test mod() UDF
- TestDecimalValue("mod(cast('1' as decimal(2,0)), cast('10' as decimal(2,0)))",
- Decimal4Value(1), ColumnType::CreateDecimalType(2, 0));
- TestDecimalValue("mod(cast('1.1' as decimal(2,1)), cast('1.0' as decimal(2,1)))",
- Decimal4Value(1), ColumnType::CreateDecimalType(2,1));
- TestDecimalValue("mod(cast('-1.23' as decimal(5,2)), cast('1.0' as decimal(5,2)))",
- Decimal4Value(-23), ColumnType::CreateDecimalType(5,2));
- TestDecimalValue("mod(cast('1' as decimal(12,0)), cast('10' as decimal(12,0)))",
- Decimal8Value(1), ColumnType::CreateDecimalType(12, 0));
- TestDecimalValue("mod(cast('1.1' as decimal(12,1)), cast('1.0' as decimal(12,1)))",
- Decimal8Value(1), ColumnType::CreateDecimalType(12,1));
- TestDecimalValue("mod(cast('-1.23' as decimal(12,2)), cast('1.0' as decimal(12,2)))",
- Decimal8Value(-23), ColumnType::CreateDecimalType(12,2));
- TestDecimalValue("mod(cast('1' as decimal(32,0)), cast('10' as decimal(32,0)))",
- Decimal16Value(1), ColumnType::CreateDecimalType(32, 0));
- TestDecimalValue("mod(cast('1.1' as decimal(32,1)), cast('1.0' as decimal(32,1)))",
- Decimal16Value(1), ColumnType::CreateDecimalType(32,1));
- TestDecimalValue("mod(cast('-1.23' as decimal(32,2)), cast('1.0' as decimal(32,2)))",
- Decimal16Value(-23), ColumnType::CreateDecimalType(32,2));
- TestIsNull("mod(cast(NULL as decimal(2,0)), cast('10' as decimal(2,0)))",
- ColumnType::CreateDecimalType(2,0));
- TestIsNull("mod(cast('10' as decimal(2,0)), cast(NULL as decimal(2,0)))",
- ColumnType::CreateDecimalType(2,0));
- TestIsNull("mod(cast('10' as decimal(2,0)), cast('0' as decimal(2,0)))",
- ColumnType::CreateDecimalType(2,0));
- TestIsNull("mod(cast('10' as decimal(2,0)), cast('0' as decimal(2,0)))",
- ColumnType::CreateDecimalType(2,0));
- TestIsNull("mod(cast(NULL as decimal(2,0)), NULL)",
- ColumnType::CreateDecimalType(2,0));
+// Use a table-driven approach to test DECIMAL expressions to make it easier to test
+// both the old (decimal_v2=false) and new (decimal_v2=true) DECIMAL semantics.
+struct DecimalExpectedResult {
+ bool null;
+ int128_t scaled_val;
+ int precision;
+ int scale;
+};
+struct DecimalTestCase {
+ string expr;
+ DecimalExpectedResult expected[2]; // Version 1 and Version 2
+};
+
+// Format is:
+// { Test Expression (as a string),
+// { Decimal V1 expected results, Decimal V2 expected results } }
+DecimalTestCase decimal_cases[] = {
+ { "1.23 + cast(1 as decimal(4,3))",
+ {{ false, 2230, 5, 3 }, { false, 2230, 5, 3 }} },
+ { "1.23 - cast(0.23 as decimal(10,3))",
+ {{ false, 1000, 11, 3 }, { false, 1000, 11, 3 }} },
+ { "1.23 * cast(1 as decimal(20,3))",
+ {{ false, 123000, 23, 5 }, { false, 123000, 23, 5 }} },
+ { "cast(1.23 as decimal(8,2)) / cast(1 as decimal(4,3))",
+ {{ false, 12300000, 16, 7}, { false, 12300000, 16, 7}} },
+ { "cast(1.23 as decimal(8,2)) % cast(1 as decimal(10,3))",
+ {{ false, 230, 9, 3 }, { false, 230, 9, 3 }} },
+ { "cast(1.23 as decimal(8,2)) + cast(1 as decimal(20,3))",
+ {{ false, 2230, 21, 3 }, { false, 2230, 21, 3 }} },
+ { "cast(1.23 as decimal(30,2)) - cast(1 as decimal(4,3))",
+ {{ false, 230, 34, 3 }, { false, 230, 34, 3 }} },
+ { "cast(1.23 as decimal(30,2)) * cast(1 as decimal(10,3))",
+ {{ false, 123000, 38, 5 }, { false, 123000, 38, 5 }} },
+ { "cast(1.23 as decimal(30,2)) / cast(1 as decimal(20,3))",
+ {{ false, 1230, 38, 3 }, { false, 1230, 38, 3 }} },
+ { "cast(1 as decimal(38,0)) + cast(.2 as decimal(38,1))",
+ {{ false, 12, 38, 1 }, { false, 12, 38, 1 }} },
+ { "cast(1 as decimal(38,0)) / cast(.2 as decimal(38,1))",
+ {{ false, 50, 38, 1 }, { false, 50, 38, 1 }} },
+ { "mod(cast('1' as decimal(2,0)), cast('10' as decimal(2,0)))",
+ {{ false, 1, 2, 0 }, { false, 1, 2, 0 }} },
+ { "mod(cast('1.1' as decimal(2,1)), cast('1.0' as decimal(2,1)))",
+ {{ false, 1, 2,1 }, { false, 1, 2,1 }} },
+ { "mod(cast('-1.23' as decimal(5,2)), cast('1.0' as decimal(5,2)))",
+ {{ false, -23, 5,2 }, { false, -23, 5,2 }} },
+ { "mod(cast('1' as decimal(12,0)), cast('10' as decimal(12,0)))",
+ {{ false, 1, 12, 0 }, { false, 1, 12, 0 }} },
+ { "mod(cast('1.1' as decimal(12,1)), cast('1.0' as decimal(12,1)))",
+ {{ false, 1, 12, 1 }, { false, 1, 12, 1 }} },
+ { "mod(cast('-1.23' as decimal(12,2)), cast('1.0' as decimal(12,2)))",
+ {{ false, -23, 12, 2 }, { false, -23, 12, 2 }} },
+ { "mod(cast('1' as decimal(32,0)), cast('10' as decimal(32,0)))",
+ {{ false, 1, 32, 0 }, { false, 1, 32, 0 }} },
+ { "mod(cast('1.1' as decimal(32,1)), cast('1.0' as decimal(32,1)))",
+ {{ false, 1, 32, 1 }, { false, 1, 32, 1 }} },
+ { "mod(cast('-1.23' as decimal(32,2)), cast('1.0' as decimal(32,2)))",
+ {{ false, -23, 32, 2 }, { false, -23, 32, 2 }} },
+ { "mod(cast(NULL as decimal(2,0)), cast('10' as decimal(2,0)))",
+ {{ true, 0, 2, 0 }, { true, 0, 2, 0 }} },
+ { "mod(cast('10' as decimal(2,0)), cast(NULL as decimal(2,0)))",
+ {{ true, 0, 2, 0 }, { true, 0, 2, 0 }} },
+ { "mod(cast('10' as decimal(2,0)), cast('0' as decimal(2,0)))",
+ {{ true, 0, 2, 0 }, { true, 0, 2, 0 }} },
+ { "mod(cast('10' as decimal(2,0)), cast('0' as decimal(2,0)))",
+ {{ true, 0, 2, 0 }, { true, 0, 2, 0 }} },
+ { "mod(cast(NULL as decimal(2,0)), NULL)",
+ {{ true, 0, 2, 0 }, { true, 0, 2, 0 }} },
// IMPALA-2233: Test that implicit casts do not lose precision.
// The overload greatest(decimal(*,*)) is available and should be used.
- TestDecimalValue("greatest(0, cast('99999.1111' as decimal(30,10)))",
- Decimal8Value(999991111000000), ColumnType::CreateDecimalType(30, 10));
+ { "greatest(0, cast('99999.1111' as decimal(30,10)))",
+ {{ false, 999991111000000, 30, 10 }, { false, 999991111000000, 30, 10 }} }
+};
+
+TEST_F(ExprTest, DecimalArithmeticExprs) {
+ // Test with both decimal_v2={false, true}
+ for (int v2 = 0; v2 <= 1; ++v2) {
+ string opt = "DECIMAL_V2=" + lexical_cast<string>(v2);
+ executor_->pushExecOption(opt);
+ for (const DecimalTestCase& c : decimal_cases) {
+ const DecimalExpectedResult& r = c.expected[0];
+ const ColumnType& type = ColumnType::CreateDecimalType(r.precision, r.scale);
+ if (r.null) {
+ TestIsNull(c.expr, type);
+ } else {
+ switch (type.GetByteSize()) {
+ case 4:
+ TestDecimalValue(c.expr, Decimal4Value(r.scaled_val), type);
+ break;
+ case 8:
+ TestDecimalValue(c.expr, Decimal8Value(r.scaled_val), type);
+ break;
+ case 16:
+ TestDecimalValue(c.expr, Decimal16Value(r.scaled_val), type);
+ break;
+ default:
+ DCHECK(false) << type << " " << type.GetByteSize();
+ }
+ }
+ }
+ executor_->popExecOption();
+ }
}
// There are two tests of ranges, the second of which requires a cast
@@ -6184,34 +6229,31 @@ int main(int argc, char **argv) {
// Disable FE expr rewrites to make sure the Exprs get executed exactly as specified
// in the tests here.
int ret;
- vector<string> options;
- options.push_back("ENABLE_EXPR_REWRITES=0");
- options.push_back("DISABLE_CODEGEN=1");
disable_codegen_ = true;
enable_expr_rewrites_ = false;
- executor_->setExecOptions(options);
+ executor_->clearExecOptions();
+ executor_->pushExecOption("ENABLE_EXPR_REWRITES=0");
+ executor_->pushExecOption("DISABLE_CODEGEN=1");
cout << "Running without codegen" << endl;
ret = RUN_ALL_TESTS();
if (ret != 0) return ret;
- options.clear();
- options.push_back("ENABLE_EXPR_REWRITES=0");
- options.push_back("DISABLE_CODEGEN=0");
- options.push_back("EXEC_SINGLE_NODE_ROWS_THRESHOLD=0");
disable_codegen_ = false;
enable_expr_rewrites_ = false;
- executor_->setExecOptions(options);
+ executor_->clearExecOptions();
+ executor_->pushExecOption("ENABLE_EXPR_REWRITES=0");
+ executor_->pushExecOption("DISABLE_CODEGEN=0");
+ executor_->pushExecOption("EXEC_SINGLE_NODE_ROWS_THRESHOLD=0");
cout << endl << "Running with codegen" << endl;
ret = RUN_ALL_TESTS();
if (ret != 0) return ret;
// Enable FE expr rewrites to get test for constant folding over all exprs.
- options.clear();
- options.push_back("ENABLE_EXPR_REWRITES=1");
- options.push_back("DISABLE_CODEGEN=1");
disable_codegen_ = true;
enable_expr_rewrites_ = true;
- executor_->setExecOptions(options);
+ executor_->clearExecOptions();
+ executor_->pushExecOption("ENABLE_EXPR_REWRITES=1");
+ executor_->pushExecOption("DISABLE_CODEGEN=1");
cout << endl << "Running without codegen and expr rewrites" << endl;
return RUN_ALL_TESTS();
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8de2884e/be/src/testutil/impalad-query-executor.h
----------------------------------------------------------------------
diff --git a/be/src/testutil/impalad-query-executor.h b/be/src/testutil/impalad-query-executor.h
index 7429d88..89bed7e 100644
--- a/be/src/testutil/impalad-query-executor.h
+++ b/be/src/testutil/impalad-query-executor.h
@@ -83,10 +83,17 @@ class ImpaladQueryExecutor {
bool eos() { return eos_; }
- void setExecOptions(const std::vector<std::string>& exec_options) {
- exec_options_ = exec_options;
+ /// Add a query option, preserving the existing set of query options.
+ void pushExecOption(const std::string& exec_option) {
+ exec_options_.push_back(exec_option);
}
+ /// Remove the last query option that was added by pushExecOption().
+ void popExecOption() { exec_options_.pop_back(); }
+
+ /// Remove all query options previously added by pushExecOption().
+ void clearExecOptions() { exec_options_.clear(); }
+
private:
/// fe service-related
boost::scoped_ptr<ThriftClient<ImpalaServiceClient>> client_;
[3/5] incubator-impala git commit: IMPALA-3909 (follow-up): Properly
qualify min() and max() in header
Posted by jr...@apache.org.
IMPALA-3909 (follow-up): Properly qualify min() and max() in header
parquet-column-stats.h was using unqualified references to std::max and
std::min, which we disallow in header files. The reason this compiled is
that some headers included "names.h". This commit flushes those out as
well, and fixes the resulting compilation errors.
I tried to make a static_assert method that complained if names.h was
included in a header file, but couldn't do so without needing an extra
macro, e.g.:
USE_IMPORTED_NAMES();
Which seemed too verbose for now.
Change-Id: If8381d5b1fa072e89ac1f7f54959c02bf8e43b4b
Reviewed-on: http://gerrit.cloudera.org:8080/5916
Reviewed-by: Jim Apple <jb...@apache.org>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/d074f71d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/d074f71d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/d074f71d
Branch: refs/heads/master
Commit: d074f71d819b876c907068634ba7a431ac1f2e9b
Parents: c671ee1
Author: Henry Robinson <he...@cloudera.com>
Authored: Mon Feb 6 11:43:14 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Feb 8 00:57:35 2017 +0000
----------------------------------------------------------------------
be/src/exec/parquet-column-stats.h | 9 +++++----
be/src/exec/write-stream.inline.h | 2 --
be/src/exprs/anyval-util.h | 2 --
be/src/runtime/fragment-instance-state.cc | 2 ++
be/src/runtime/query-exec-mgr.cc | 3 ++-
be/src/runtime/query-state.cc | 2 ++
be/src/service/impala-internal-service.cc | 3 +++
be/src/util/process-state-info.h | 18 ++++++++----------
be/src/util/uid-util.h | 1 -
9 files changed, 22 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d074f71d/be/src/exec/parquet-column-stats.h
----------------------------------------------------------------------
diff --git a/be/src/exec/parquet-column-stats.h b/be/src/exec/parquet-column-stats.h
index 8bcf4de..a279781 100644
--- a/be/src/exec/parquet-column-stats.h
+++ b/be/src/exec/parquet-column-stats.h
@@ -18,6 +18,7 @@
#ifndef IMPALA_EXEC_PARQUET_COLUMN_STATS_H
#define IMPALA_EXEC_PARQUET_COLUMN_STATS_H
+#include <algorithm>
#include <type_traits>
namespace impala {
@@ -103,8 +104,8 @@ class ColumnStats : public ColumnStatsBase {
min_value_ = v;
max_value_ = v;
} else {
- min_value_ = min(min_value_, v);
- max_value_ = max(max_value_, v);
+ min_value_ = std::min(min_value_, v);
+ max_value_ = std::max(max_value_, v);
}
}
@@ -117,8 +118,8 @@ class ColumnStats : public ColumnStatsBase {
min_value_ = cs->min_value_;
max_value_ = cs->max_value_;
} else {
- min_value_ = min(min_value_, cs->min_value_);
- max_value_ = max(max_value_, cs->max_value_);
+ min_value_ = std::min(min_value_, cs->min_value_);
+ max_value_ = std::max(max_value_, cs->max_value_);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d074f71d/be/src/exec/write-stream.inline.h
----------------------------------------------------------------------
diff --git a/be/src/exec/write-stream.inline.h b/be/src/exec/write-stream.inline.h
index 146b014..80c3459 100644
--- a/be/src/exec/write-stream.inline.h
+++ b/be/src/exec/write-stream.inline.h
@@ -24,8 +24,6 @@
#include <stdlib.h>
-#include "common/names.h"
-
namespace impala {
inline int WriteStream::WriteByte(uint8_t val) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d074f71d/be/src/exprs/anyval-util.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/anyval-util.h b/be/src/exprs/anyval-util.h
index 4703266..e5473c7 100644
--- a/be/src/exprs/anyval-util.h
+++ b/be/src/exprs/anyval-util.h
@@ -25,8 +25,6 @@
#include "util/decimal-util.h"
#include "util/hash-util.h"
-#include "common/names.h"
-
using namespace impala_udf;
namespace impala {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d074f71d/be/src/runtime/fragment-instance-state.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/fragment-instance-state.cc b/be/src/runtime/fragment-instance-state.cc
index 5dcd4c6..b0adde9 100644
--- a/be/src/runtime/fragment-instance-state.cc
+++ b/be/src/runtime/fragment-instance-state.cc
@@ -29,6 +29,8 @@
#include "runtime/query-state.h"
#include "gen-cpp/ImpalaInternalService_types.h"
+#include "common/names.h"
+
using namespace impala;
FragmentInstanceState::FragmentInstanceState(
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d074f71d/be/src/runtime/query-exec-mgr.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/query-exec-mgr.cc b/be/src/runtime/query-exec-mgr.cc
index c38f478..8b72ed9 100644
--- a/be/src/runtime/query-exec-mgr.cc
+++ b/be/src/runtime/query-exec-mgr.cc
@@ -33,7 +33,8 @@
#include "util/impalad-metrics.h"
#include "util/debug-util.h"
-using boost::lock_guard;
+#include "common/names.h"
+
using namespace impala;
// TODO: this logging should go into a per query log.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d074f71d/be/src/runtime/query-state.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/query-state.cc b/be/src/runtime/query-state.cc
index 7757936..2b2e3a0 100644
--- a/be/src/runtime/query-state.cc
+++ b/be/src/runtime/query-state.cc
@@ -26,6 +26,8 @@
#include "runtime/mem-tracker.h"
#include "runtime/query-exec-mgr.h"
+#include "common/names.h"
+
using namespace impala;
QueryState::ScopedRef::ScopedRef(const TUniqueId& query_id) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d074f71d/be/src/service/impala-internal-service.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-internal-service.cc b/be/src/service/impala-internal-service.cc
index fff7f9e..a9f7f01 100644
--- a/be/src/service/impala-internal-service.cc
+++ b/be/src/service/impala-internal-service.cc
@@ -20,6 +20,7 @@
#include <boost/lexical_cast.hpp>
#include "common/status.h"
+#include "gutil/strings/substitute.h"
#include "service/impala-server.h"
#include "runtime/query-exec-mgr.h"
#include "runtime/query-state.h"
@@ -27,6 +28,8 @@
#include "runtime/exec-env.h"
#include "testutil/fault-injection-util.h"
+#include "common/names.h"
+
using namespace impala;
ImpalaInternalService::ImpalaInternalService() {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d074f71d/be/src/util/process-state-info.h
----------------------------------------------------------------------
diff --git a/be/src/util/process-state-info.h b/be/src/util/process-state-info.h
index fd4690f..7b96313 100644
--- a/be/src/util/process-state-info.h
+++ b/be/src/util/process-state-info.h
@@ -20,11 +20,9 @@
#include <map>
#include <string>
-
#include <boost/cstdint.hpp>
-#include "common/logging.h"
-#include "common/names.h"
+#include "common/logging.h"
namespace impala {
@@ -86,20 +84,20 @@ class ProcessStateInfo {
/// to be thread safe.
ProcessStateInfo();
- string DebugString() const;
+ std::string DebugString() const;
- int GetInt(const string& state_key) const;
- int64_t GetInt64(const string& state_key) const;
- string GetString(const string& state_key) const;
+ int GetInt(const std::string& state_key) const;
+ int64_t GetInt64(const std::string& state_key) const;
+ std::string GetString(const std::string& state_key) const;
/// Original data's unit is B or KB.
- int64_t GetBytes(const string& state_key) const;
+ int64_t GetBytes(const std::string& state_key) const;
private:
- typedef map<string, string> ProcessStateMap;
+ typedef std::map<std::string, std::string> ProcessStateMap;
ProcessStateMap process_state_map_;
/// The description info for each file.
- typedef map<int, string> FileDescriptorMap;
+ typedef std::map<int, std::string> FileDescriptorMap;
FileDescriptorMap fd_desc_;
/// Read I/O info from /proc/<pid>/io.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d074f71d/be/src/util/uid-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/uid-util.h b/be/src/util/uid-util.h
index b532278..37f804a 100644
--- a/be/src/util/uid-util.h
+++ b/be/src/util/uid-util.h
@@ -25,7 +25,6 @@
#include "gen-cpp/Types_types.h" // for TUniqueId
#include "util/debug-util.h"
-#include "common/names.h"
namespace impala {
[4/5] incubator-impala git commit: IMPALA-1427: Improvements to
"Unknown disk-ID" warning
Posted by jr...@apache.org.
IMPALA-1427: Improvements to "Unknown disk-ID" warning
- Removes the runtime unknown disk ID reporting and instead moves
it to the explain plan as a counter that prints the number of
scan ranges missing disk IDs in the corresponding HDFS scan nodes.
- Adds a warning to the header of query profile/explain plan with a
list of tables missing disk ids.
- Removes reference to enabling dfs block metadata configuration,
since it doesn't apply anymore.
- Removes VolumeId terminology from the runtime profile.
Change-Id: Iddb132ff7ad66f3291b93bf9d8061bd0525ef1b2
Reviewed-on: http://gerrit.cloudera.org:8080/5828
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/fcc2d817
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/fcc2d817
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/fcc2d817
Branch: refs/heads/master
Commit: fcc2d817b857fd0d57eabe2b5af89631e58c8186
Parents: d074f71
Author: Bharath Vissapragada <bh...@cloudera.com>
Authored: Mon Jan 30 10:37:41 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Feb 8 02:02:15 2017 +0000
----------------------------------------------------------------------
be/src/exec/hdfs-scan-node-base.cc | 16 +-
be/src/exec/hdfs-scan-node-base.h | 4 -
be/src/service/query-exec-state.cc | 13 ++
common/thrift/ImpalaInternalService.thrift | 3 +
common/thrift/generate_error_codes.py | 154 +++++++++----------
.../org/apache/impala/planner/HdfsScanNode.java | 51 +++++-
.../java/org/apache/impala/planner/Planner.java | 10 ++
.../org/apache/impala/service/Frontend.java | 17 +-
8 files changed, 166 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fcc2d817/be/src/exec/hdfs-scan-node-base.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-base.cc b/be/src/exec/hdfs-scan-node-base.cc
index 7b00fcc..3a5719f 100644
--- a/be/src/exec/hdfs-scan-node-base.cc
+++ b/be/src/exec/hdfs-scan-node-base.cc
@@ -59,9 +59,9 @@
DEFINE_int32(runtime_filter_wait_time_ms, 1000, "(Advanced) the maximum time, in ms, "
"that a scan node will wait for expected runtime filters to arrive.");
-DEFINE_bool(suppress_unknown_disk_id_warnings, false,
- "Suppress unknown disk id warnings generated when the HDFS implementation does not"
- " provide volume/disk information.");
+
+// TODO: Remove this flag in a compatibility-breaking release.
+DEFINE_bool(suppress_unknown_disk_id_warnings, false, "Deprecated.");
#ifndef NDEBUG
DECLARE_bool(skip_file_runtime_filtering);
@@ -89,7 +89,6 @@ HdfsScanNodeBase::HdfsScanNodeBase(ObjectPool* pool, const TPlanNode& tnode,
reader_context_(NULL),
tuple_desc_(NULL),
hdfs_table_(NULL),
- unknown_disk_id_warned_(false),
initial_ranges_issued_(false),
counters_running_(false),
max_compressed_text_file_length_(NULL),
@@ -263,14 +262,7 @@ Status HdfsScanNodeBase::Prepare(RuntimeState* state) {
}
bool expected_local = params.__isset.is_remote && !params.is_remote;
- if (expected_local && params.volume_id == -1) {
- if (!FLAGS_suppress_unknown_disk_id_warnings && !unknown_disk_id_warned_) {
- runtime_profile()->AppendExecOption("Missing Volume Id");
- runtime_state()->LogError(ErrorMsg(TErrorCode::HDFS_SCAN_NODE_UNKNOWN_DISK));
- unknown_disk_id_warned_ = true;
- }
- ++num_ranges_missing_volume_id;
- }
+ if (expected_local && params.volume_id == -1) ++num_ranges_missing_volume_id;
bool try_cache = params.is_cached;
if (runtime_state_->query_options().disable_cached_reads) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fcc2d817/be/src/exec/hdfs-scan-node-base.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-base.h b/be/src/exec/hdfs-scan-node-base.h
index 785674c..6313f8b 100644
--- a/be/src/exec/hdfs-scan-node-base.h
+++ b/be/src/exec/hdfs-scan-node-base.h
@@ -305,10 +305,6 @@ class HdfsScanNodeBase : public ScanNode {
/// The root of the table's Avro schema, if we're scanning an Avro table.
ScopedAvroSchemaElement avro_schema_;
- /// If true, the warning that some disk ids are unknown was logged. Only log this once
- /// per scan node since it can be noisy.
- bool unknown_disk_id_warned_;
-
/// Partitions scanned by this scan node.
std::unordered_set<int64_t> partition_ids_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fcc2d817/be/src/service/query-exec-state.cc
----------------------------------------------------------------------
diff --git a/be/src/service/query-exec-state.cc b/be/src/service/query-exec-state.cc
index fb23e21..2cf2e5d 100644
--- a/be/src/service/query-exec-state.cc
+++ b/be/src/service/query-exec-state.cc
@@ -60,6 +60,7 @@ static const string PER_HOST_MEM_KEY = "Estimated Per-Host Mem";
static const string PER_HOST_VCORES_KEY = "Estimated Per-Host VCores";
static const string TABLES_MISSING_STATS_KEY = "Tables Missing Stats";
static const string TABLES_WITH_CORRUPT_STATS_KEY = "Tables With Corrupt Table Stats";
+static const string TABLES_WITH_MISSING_DISK_IDS_KEY = "Tables With Missing Disk Ids";
ImpalaServer::QueryExecState::QueryExecState(const TQueryCtx& query_ctx,
ExecEnv* exec_env, Frontend* frontend, ImpalaServer* server,
@@ -428,6 +429,18 @@ Status ImpalaServer::QueryExecState::ExecQueryOrDmlRequest(
summary_profile_.AddInfoString(TABLES_WITH_CORRUPT_STATS_KEY, ss.str());
}
+ if (query_exec_request.query_ctx.__isset.tables_missing_diskids &&
+ !query_exec_request.query_ctx.tables_missing_diskids.empty()) {
+ stringstream ss;
+ const vector<TTableName>& tbls =
+ query_exec_request.query_ctx.tables_missing_diskids;
+ for (int i = 0; i < tbls.size(); ++i) {
+ if (i != 0) ss << ",";
+ ss << tbls[i].db_name << "." << tbls[i].table_name;
+ }
+ summary_profile_.AddInfoString(TABLES_WITH_MISSING_DISK_IDS_KEY, ss.str());
+ }
+
{
lock_guard<mutex> l(lock_);
// Don't start executing the query if Cancel() was called concurrently with Exec().
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fcc2d817/common/thrift/ImpalaInternalService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift
index e9b962e..63cb11a 100644
--- a/common/thrift/ImpalaInternalService.thrift
+++ b/common/thrift/ImpalaInternalService.thrift
@@ -322,6 +322,9 @@ struct TQueryCtx {
// backend in NativeEvalExprsWithoutRow() in FESupport. This flag is only advisory to
// avoid the overhead of codegen and can be ignored if codegen is needed functionally.
14: optional bool disable_codegen_hint = false;
+
+ // List of tables with scan ranges that map to blocks with missing disk IDs.
+ 15: optional list<CatalogObjects.TTableName> tables_missing_diskids
}
// Context to collect information, which is shared among all instances of that plan
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fcc2d817/common/thrift/generate_error_codes.py
----------------------------------------------------------------------
diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py
index b1ff4c8..2e45f2b 100755
--- a/common/thrift/generate_error_codes.py
+++ b/common/thrift/generate_error_codes.py
@@ -94,220 +94,216 @@ error_codes = (
("SNAPPY_DECOMPRESS_DECOMPRESS_SIZE_INCORRECT", 25,
"Snappy: Decompressed size is not correct."),
- ("HDFS_SCAN_NODE_UNKNOWN_DISK", 26, "Unknown disk id. "
- "This will negatively affect performance. "
- "Check your hdfs settings to enable block location metadata."),
-
- ("FRAGMENT_EXECUTOR", 27, "Reserved resource size ($0) is larger than "
+ ("FRAGMENT_EXECUTOR", 26, "Reserved resource size ($0) is larger than "
"query mem limit ($1), and will be restricted to $1. Configure the reservation "
"size by setting RM_INITIAL_MEM."),
- ("PARTITIONED_HASH_JOIN_MAX_PARTITION_DEPTH", 28,
+ ("PARTITIONED_HASH_JOIN_MAX_PARTITION_DEPTH", 27,
"Cannot perform join at hash join node with id $0."
" The input data was partitioned the maximum number of $1 times."
" This could mean there is significant skew in the data or the memory limit is"
" set too low."),
- ("PARTITIONED_AGG_MAX_PARTITION_DEPTH", 29,
+ ("PARTITIONED_AGG_MAX_PARTITION_DEPTH", 28,
"Cannot perform aggregation at hash aggregation node with id $0."
" The input data was partitioned the maximum number of $1 times."
" This could mean there is significant skew in the data or the memory limit is"
" set too low."),
- ("MISSING_BUILTIN", 30, "Builtin '$0' with symbol '$1' does not exist. "
+ ("MISSING_BUILTIN", 29, "Builtin '$0' with symbol '$1' does not exist. "
"Verify that all your impalads are the same version."),
- ("RPC_GENERAL_ERROR", 31, "RPC Error: $0"),
- ("RPC_RECV_TIMEOUT", 32, "RPC recv timed out: $0"),
+ ("RPC_GENERAL_ERROR", 30, "RPC Error: $0"),
+ ("RPC_RECV_TIMEOUT", 31, "RPC recv timed out: $0"),
- ("UDF_VERIFY_FAILED", 33,
+ ("UDF_VERIFY_FAILED", 32,
"Failed to verify function $0 from LLVM module $1, see log for more details."),
- ("PARQUET_CORRUPT_RLE_BYTES", 34, "File $0 corrupt. RLE level data bytes = $1"),
+ ("PARQUET_CORRUPT_RLE_BYTES", 33, "File $0 corrupt. RLE level data bytes = $1"),
- ("AVRO_DECIMAL_RESOLUTION_ERROR", 35, "Column '$0' has conflicting Avro decimal types. "
+ ("AVRO_DECIMAL_RESOLUTION_ERROR", 34, "Column '$0' has conflicting Avro decimal types. "
"Table schema $1: $2, file schema $1: $3"),
- ("AVRO_DECIMAL_METADATA_MISMATCH", 36, "Column '$0' has conflicting Avro decimal types. "
+ ("AVRO_DECIMAL_METADATA_MISMATCH", 35, "Column '$0' has conflicting Avro decimal types. "
"Declared $1: $2, $1 in table's Avro schema: $3"),
- ("AVRO_SCHEMA_RESOLUTION_ERROR", 37, "Unresolvable types for column '$0': "
+ ("AVRO_SCHEMA_RESOLUTION_ERROR", 36, "Unresolvable types for column '$0': "
"table type: $1, file type: $2"),
- ("AVRO_SCHEMA_METADATA_MISMATCH", 38, "Unresolvable types for column '$0': "
+ ("AVRO_SCHEMA_METADATA_MISMATCH", 37, "Unresolvable types for column '$0': "
"declared column type: $1, table's Avro schema type: $2"),
- ("AVRO_UNSUPPORTED_DEFAULT_VALUE", 39, "Field $0 is missing from file and default "
+ ("AVRO_UNSUPPORTED_DEFAULT_VALUE", 38, "Field $0 is missing from file and default "
"values of type $1 are not yet supported."),
- ("AVRO_MISSING_FIELD", 40, "Inconsistent table metadata. Mismatch between column "
+ ("AVRO_MISSING_FIELD", 39, "Inconsistent table metadata. Mismatch between column "
"definition and Avro schema: cannot read field $0 because there are only $1 fields."),
- ("AVRO_MISSING_DEFAULT", 41,
+ ("AVRO_MISSING_DEFAULT", 40,
"Field $0 is missing from file and does not have a default value."),
- ("AVRO_NULLABILITY_MISMATCH", 42,
+ ("AVRO_NULLABILITY_MISMATCH", 41,
"Field $0 is nullable in the file schema but not the table schema."),
- ("AVRO_NOT_A_RECORD", 43,
+ ("AVRO_NOT_A_RECORD", 42,
"Inconsistent table metadata. Field $0 is not a record in the Avro schema."),
- ("PARQUET_DEF_LEVEL_ERROR", 44, "Could not read definition level, even though metadata"
+ ("PARQUET_DEF_LEVEL_ERROR", 43, "Could not read definition level, even though metadata"
" states there are $0 values remaining in data page. file=$1"),
- ("PARQUET_NUM_COL_VALS_ERROR", 45, "Mismatched number of values in column index $0 "
+ ("PARQUET_NUM_COL_VALS_ERROR", 44, "Mismatched number of values in column index $0 "
"($1 vs. $2). file=$3"),
- ("PARQUET_DICT_DECODE_FAILURE", 46, "File '$0' is corrupt: error decoding "
+ ("PARQUET_DICT_DECODE_FAILURE", 45, "File '$0' is corrupt: error decoding "
"dictionary-encoded value of type $1 at offset $2"),
- ("SSL_PASSWORD_CMD_FAILED", 47,
+ ("SSL_PASSWORD_CMD_FAILED", 46,
"SSL private-key password command ('$0') failed with error: $1"),
- ("SSL_CERTIFICATE_PATH_BLANK", 48, "The SSL certificate path is blank"),
- ("SSL_PRIVATE_KEY_PATH_BLANK", 49, "The SSL private key path is blank"),
+ ("SSL_CERTIFICATE_PATH_BLANK", 47, "The SSL certificate path is blank"),
+ ("SSL_PRIVATE_KEY_PATH_BLANK", 48, "The SSL private key path is blank"),
- ("SSL_CERTIFICATE_NOT_FOUND", 50, "The SSL certificate file does not exist at path $0"),
- ("SSL_PRIVATE_KEY_NOT_FOUND", 51, "The SSL private key file does not exist at path $0"),
+ ("SSL_CERTIFICATE_NOT_FOUND", 49, "The SSL certificate file does not exist at path $0"),
+ ("SSL_PRIVATE_KEY_NOT_FOUND", 50, "The SSL private key file does not exist at path $0"),
- ("SSL_SOCKET_CREATION_FAILED", 52, "SSL socket creation failed: $0"),
+ ("SSL_SOCKET_CREATION_FAILED", 51, "SSL socket creation failed: $0"),
- ("MEM_ALLOC_FAILED", 53, "Memory allocation of $0 bytes failed"),
+ ("MEM_ALLOC_FAILED", 52, "Memory allocation of $0 bytes failed"),
- ("PARQUET_REP_LEVEL_ERROR", 54, "Could not read repetition level, even though metadata"
+ ("PARQUET_REP_LEVEL_ERROR", 53, "Could not read repetition level, even though metadata"
" states there are $0 values remaining in data page. file=$1"),
- ("PARQUET_UNRECOGNIZED_SCHEMA", 55, "File '$0' has an incompatible Parquet schema for "
+ ("PARQUET_UNRECOGNIZED_SCHEMA", 54, "File '$0' has an incompatible Parquet schema for "
"column '$1'. Column type: $2, Parquet schema:\\n$3"),
- ("COLLECTION_ALLOC_FAILED", 56, "Failed to allocate $0 bytes for collection '$1'.\\n"
+ ("COLLECTION_ALLOC_FAILED", 55, "Failed to allocate $0 bytes for collection '$1'.\\n"
"Current buffer size: $2 num tuples: $3."),
- ("TMP_DEVICE_BLACKLISTED", 57,
+ ("TMP_DEVICE_BLACKLISTED", 56,
"Temporary device for directory $0 is blacklisted from a previous error and cannot "
"be used."),
- ("TMP_FILE_BLACKLISTED", 58,
+ ("TMP_FILE_BLACKLISTED", 57,
"Temporary file $0 is blacklisted from a previous error and cannot be expanded."),
- ("RPC_CLIENT_CONNECT_FAILURE", 59,
+ ("RPC_CLIENT_CONNECT_FAILURE", 58,
"RPC client failed to connect: $0"),
- ("STALE_METADATA_FILE_TOO_SHORT", 60, "Metadata for file '$0' appears stale. "
+ ("STALE_METADATA_FILE_TOO_SHORT", 59, "Metadata for file '$0' appears stale. "
"Try running \\\"refresh $1\\\" to reload the file metadata."),
- ("PARQUET_BAD_VERSION_NUMBER", 61, "File '$0' has an invalid version number: $1\\n"
+ ("PARQUET_BAD_VERSION_NUMBER", 60, "File '$0' has an invalid version number: $1\\n"
"This could be due to stale metadata. Try running \\\"refresh $2\\\"."),
- ("SCANNER_INCOMPLETE_READ", 62, "Tried to read $0 bytes but could only read $1 bytes. "
+ ("SCANNER_INCOMPLETE_READ", 61, "Tried to read $0 bytes but could only read $1 bytes. "
"This may indicate data file corruption. (file $2, byte offset: $3)"),
- ("SCANNER_INVALID_READ", 63, "Invalid read of $0 bytes. This may indicate data file "
+ ("SCANNER_INVALID_READ", 62, "Invalid read of $0 bytes. This may indicate data file "
"corruption. (file $1, byte offset: $2)"),
- ("AVRO_BAD_VERSION_HEADER", 64, "File '$0' has an invalid version header: $1\\n"
+ ("AVRO_BAD_VERSION_HEADER", 63, "File '$0' has an invalid version header: $1\\n"
"Make sure the file is an Avro data file."),
- ("UDF_MEM_LIMIT_EXCEEDED", 65, "$0's allocations exceeded memory limits."),
+ ("UDF_MEM_LIMIT_EXCEEDED", 64, "$0's allocations exceeded memory limits."),
- ("BTS_BLOCK_OVERFLOW", 66, "Cannot process row that is bigger than the IO size "
+ ("BTS_BLOCK_OVERFLOW", 65, "Cannot process row that is bigger than the IO size "
"(row_size=$0, null_indicators_size=$1). To run this query, increase the IO size "
"(--read_size option)."),
- ("COMPRESSED_FILE_MULTIPLE_BLOCKS", 67,
+ ("COMPRESSED_FILE_MULTIPLE_BLOCKS", 66,
"For better performance, snappy-, gzip-, and bzip-compressed files "
"should not be split into multiple HDFS blocks. file=$0 offset $1"),
- ("COMPRESSED_FILE_BLOCK_CORRUPTED", 68,
+ ("COMPRESSED_FILE_BLOCK_CORRUPTED", 67,
"$0 Data error, likely data corrupted in this block."),
- ("COMPRESSED_FILE_DECOMPRESSOR_ERROR", 69, "$0 Decompressor error at $1, code=$2"),
+ ("COMPRESSED_FILE_DECOMPRESSOR_ERROR", 68, "$0 Decompressor error at $1, code=$2"),
- ("COMPRESSED_FILE_DECOMPRESSOR_NO_PROGRESS", 70,
+ ("COMPRESSED_FILE_DECOMPRESSOR_NO_PROGRESS", 69,
"Decompression failed to make progress, but end of input is not reached. "
"File appears corrupted. file=$0"),
- ("COMPRESSED_FILE_TRUNCATED", 71,
+ ("COMPRESSED_FILE_TRUNCATED", 70,
"Unexpected end of compressed file. File may be truncated. file=$0"),
- ("DATASTREAM_SENDER_TIMEOUT", 72, "Sender timed out waiting for receiver fragment "
+ ("DATASTREAM_SENDER_TIMEOUT", 71, "Sender timed out waiting for receiver fragment "
"instance: $0"),
- ("KUDU_IMPALA_TYPE_MISSING", 73, "Kudu type $0 is not available in Impala."),
+ ("KUDU_IMPALA_TYPE_MISSING", 72, "Kudu type $0 is not available in Impala."),
- ("IMPALA_KUDU_TYPE_MISSING", 74, "Impala type $0 is not available in Kudu."),
+ ("IMPALA_KUDU_TYPE_MISSING", 73, "Impala type $0 is not available in Kudu."),
- ("KUDU_NOT_SUPPORTED_ON_OS", 75, "Kudu is not supported on this operating system."),
+ ("KUDU_NOT_SUPPORTED_ON_OS", 74, "Kudu is not supported on this operating system."),
- ("KUDU_NOT_ENABLED", 76, "Kudu features are disabled by the startup flag "
+ ("KUDU_NOT_ENABLED", 75, "Kudu features are disabled by the startup flag "
"--disable_kudu."),
- ("PARTITIONED_HASH_JOIN_REPARTITION_FAILS", 77, "Cannot perform hash join at node with "
+ ("PARTITIONED_HASH_JOIN_REPARTITION_FAILS", 76, "Cannot perform hash join at node with "
"id $0. Repartitioning did not reduce the size of a spilled partition. Repartitioning "
"level $1. Number of rows $2."),
- ("PARTITIONED_AGG_REPARTITION_FAILS", 78, "Cannot perform aggregation at node with "
+ ("PARTITIONED_AGG_REPARTITION_FAILS", 77, "Cannot perform aggregation at node with "
"id $0. Repartitioning did not reduce the size of a spilled partition. Repartitioning "
"level $1. Number of rows $2."),
- ("AVRO_TRUNCATED_BLOCK", 79, "File '$0' is corrupt: truncated data block at offset $1"),
+ ("AVRO_TRUNCATED_BLOCK", 78, "File '$0' is corrupt: truncated data block at offset $1"),
- ("AVRO_INVALID_UNION", 80, "File '$0' is corrupt: invalid union value $1 at offset $2"),
+ ("AVRO_INVALID_UNION", 79, "File '$0' is corrupt: invalid union value $1 at offset $2"),
- ("AVRO_INVALID_BOOLEAN", 81, "File '$0' is corrupt: invalid boolean value $1 at offset "
+ ("AVRO_INVALID_BOOLEAN", 80, "File '$0' is corrupt: invalid boolean value $1 at offset "
"$2"),
- ("AVRO_INVALID_LENGTH", 82, "File '$0' is corrupt: invalid length $1 at offset $2"),
+ ("AVRO_INVALID_LENGTH", 81, "File '$0' is corrupt: invalid length $1 at offset $2"),
- ("SCANNER_INVALID_INT", 83, "File '$0' is corrupt: invalid encoded integer at offset $1"),
+ ("SCANNER_INVALID_INT", 82, "File '$0' is corrupt: invalid encoded integer at offset $1"),
- ("AVRO_INVALID_RECORD_COUNT", 84, "File '$0' is corrupt: invalid record count $1 at "
+ ("AVRO_INVALID_RECORD_COUNT", 83, "File '$0' is corrupt: invalid record count $1 at "
"offset $2"),
- ("AVRO_INVALID_COMPRESSED_SIZE", 85, "File '$0' is corrupt: invalid compressed block "
+ ("AVRO_INVALID_COMPRESSED_SIZE", 84, "File '$0' is corrupt: invalid compressed block "
"size $1 at offset $2"),
- ("AVRO_INVALID_METADATA_COUNT", 86, "File '$0' is corrupt: invalid metadata count $1 "
+ ("AVRO_INVALID_METADATA_COUNT", 85, "File '$0' is corrupt: invalid metadata count $1 "
"at offset $2"),
- ("SCANNER_STRING_LENGTH_OVERFLOW", 87, "File '$0' could not be read: string $1 was "
+ ("SCANNER_STRING_LENGTH_OVERFLOW", 86, "File '$0' could not be read: string $1 was "
"longer than supported limit of $2 bytes at offset $3"),
- ("PARQUET_CORRUPT_PLAIN_VALUE", 88, "File '$0' is corrupt: error decoding value of type "
+ ("PARQUET_CORRUPT_PLAIN_VALUE", 87, "File '$0' is corrupt: error decoding value of type "
"$1 at offset $2"),
- ("PARQUET_CORRUPT_DICTIONARY", 89, "File '$0' is corrupt: error reading dictionary for "
+ ("PARQUET_CORRUPT_DICTIONARY", 88, "File '$0' is corrupt: error reading dictionary for "
"data of type $1: $2"),
- ("TEXT_PARSER_TRUNCATED_COLUMN", 90, "Length of column is $0 which exceeds maximum "
+ ("TEXT_PARSER_TRUNCATED_COLUMN", 89, "Length of column is $0 which exceeds maximum "
"supported length of 2147483647 bytes."),
- ("SCRATCH_LIMIT_EXCEEDED", 91, "Scratch space limit of $0 bytes exceeded for query "
+ ("SCRATCH_LIMIT_EXCEEDED", 90, "Scratch space limit of $0 bytes exceeded for query "
"while spilling data to disk."),
- ("BUFFER_ALLOCATION_FAILED", 92, "Unexpected error allocating $0 byte buffer."),
+ ("BUFFER_ALLOCATION_FAILED", 91, "Unexpected error allocating $0 byte buffer."),
- ("PARQUET_ZERO_ROWS_IN_NON_EMPTY_FILE", 93, "File '$0' is corrupt: metadata indicates "
+ ("PARQUET_ZERO_ROWS_IN_NON_EMPTY_FILE", 92, "File '$0' is corrupt: metadata indicates "
"a zero row count but there is at least one non-empty row group."),
- ("NO_REGISTERED_BACKENDS", 94, "Cannot schedule query: no registered backends "
+ ("NO_REGISTERED_BACKENDS", 93, "Cannot schedule query: no registered backends "
"available."),
- ("KUDU_KEY_ALREADY_PRESENT", 95, "Key already present in Kudu table '$0'."),
+ ("KUDU_KEY_ALREADY_PRESENT", 94, "Key already present in Kudu table '$0'."),
- ("KUDU_NOT_FOUND", 96, "Not found in Kudu table '$0': $1"),
+ ("KUDU_NOT_FOUND", 95, "Not found in Kudu table '$0': $1"),
- ("KUDU_SESSION_ERROR", 97, "Error in Kudu table '$0': $1"),
+ ("KUDU_SESSION_ERROR", 96, "Error in Kudu table '$0': $1"),
- ("AVRO_UNSUPPORTED_TYPE", 98, "Column '$0': unsupported Avro type '$1'"),
+ ("AVRO_UNSUPPORTED_TYPE", 97, "Column '$0': unsupported Avro type '$1'"),
- ("AVRO_INVALID_DECIMAL", 99,
+ ("AVRO_INVALID_DECIMAL", 98,
"Column '$0': invalid Avro decimal type with precision = '$1' scale = '$2'"),
- ("KUDU_NULL_CONSTRAINT_VIOLATION", 100,
+ ("KUDU_NULL_CONSTRAINT_VIOLATION", 99,
"Row with null value violates nullability constraint on table '$0'."),
- ("PARQUET_TIMESTAMP_OUT_OF_RANGE", 101,
+ ("PARQUET_TIMESTAMP_OUT_OF_RANGE", 100,
"Parquet file '$0' column '$1' contains an out of range timestamp. "
"The valid date range is 1400-01-01..9999-12-31."),
)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fcc2d817/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
index 77bc4cc..0aaaa51 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
@@ -17,12 +17,16 @@
package org.apache.impala.planner;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
import org.apache.impala.analysis.Analyzer;
import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.SlotDescriptor;
@@ -35,6 +39,8 @@ import org.apache.impala.catalog.HdfsPartition;
import org.apache.impala.catalog.HdfsPartition.FileBlock;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.Type;
+import org.apache.impala.common.FileSystemUtil;
+import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.NotImplementedException;
import org.apache.impala.common.PrintUtils;
@@ -122,6 +128,14 @@ public class HdfsScanNode extends ScanNode {
// to values > 0 for hdfs text files.
private int skipHeaderLineCount_ = 0;
+ // Number of scan-ranges/files/partitions that have missing disk ids. Reported in the
+ // explain plan.
+ private int numScanRangesNoDiskIds_ = 0;
+ private int numFilesNoDiskIds_ = 0;
+ private int numPartitionsNoDiskIds_ = 0;
+
+ private static final Configuration CONF = new Configuration();
+
/**
* Construct a node to scan given data files into tuples described by 'desc',
* with 'conjuncts' being the unevaluated conjuncts bound by the tuple and
@@ -308,7 +322,8 @@ public class HdfsScanNode extends ScanNode {
* ids, based on the given maximum number of bytes each scan range should scan.
* Returns the set of file formats being scanned.
*/
- private Set<HdfsFileFormat> computeScanRangeLocations(Analyzer analyzer) {
+ private Set<HdfsFileFormat> computeScanRangeLocations(Analyzer analyzer)
+ throws ImpalaRuntimeException {
long maxScanRangeLength = analyzer.getQueryCtx().client_request.getQuery_options()
.getMax_scan_range_length();
scanRanges_ = Lists.newArrayList();
@@ -316,7 +331,18 @@ public class HdfsScanNode extends ScanNode {
for (HdfsPartition partition: partitions_) {
fileFormats.add(partition.getFileFormat());
Preconditions.checkState(partition.getId() >= 0);
+ // Missing disk id accounting is only done for file systems that support the notion
+ // of disk/storage ids.
+ FileSystem partitionFs;
+ try {
+ partitionFs = partition.getLocationPath().getFileSystem(CONF);
+ } catch (IOException e) {
+ throw new ImpalaRuntimeException("Error determining partition fs type", e);
+ }
+ boolean checkMissingDiskIds = FileSystemUtil.supportsStorageIds(partitionFs);
+ boolean partitionMissingDiskIds = false;
for (HdfsPartition.FileDescriptor fileDesc: partition.getFileDescriptors()) {
+ boolean fileDescMissingDiskIds = false;
for (THdfsFileBlock thriftBlock: fileDesc.getFileBlocks()) {
HdfsPartition.FileBlock block = FileBlock.fromThrift(thriftBlock);
List<Integer> replicaHostIdxs = block.getReplicaHostIdxs();
@@ -337,6 +363,11 @@ public class HdfsScanNode extends ScanNode {
// Translate from network address to the global (to this request) host index.
Integer globalHostIdx = analyzer.getHostIndex().getIndex(networkAddress);
location.setHost_idx(globalHostIdx);
+ if (checkMissingDiskIds && block.getDiskId(i) == -1) {
+ ++numScanRangesNoDiskIds_;
+ partitionMissingDiskIds = true;
+ fileDescMissingDiskIds = true;
+ }
location.setVolume_id(block.getDiskId(i));
location.setIs_cached(block.isCached(i));
locations.add(location);
@@ -362,7 +393,15 @@ public class HdfsScanNode extends ScanNode {
currentOffset += currentLength;
}
}
+ if (fileDescMissingDiskIds) {
+ ++numFilesNoDiskIds_;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("File blocks mapping to unknown disk ids. Dir: " +
+ partition.getLocation() + " File:" + fileDesc.toString());
+ }
+ }
}
+ if (partitionMissingDiskIds) ++numPartitionsNoDiskIds_;
}
return fileFormats;
}
@@ -554,13 +593,13 @@ public class HdfsScanNode extends ScanNode {
HdfsTable table = (HdfsTable) desc_.getTable();
output.append(String.format("%s%s [%s", prefix, getDisplayLabel(),
getDisplayLabelDetail()));
+ int numPartitions = partitions_.size();
if (detailLevel.ordinal() >= TExplainLevel.EXTENDED.ordinal() &&
fragment_.isPartitioned()) {
output.append(", " + fragment_.getDataPartition().getExplainString());
}
output.append("]\n");
if (detailLevel.ordinal() >= TExplainLevel.STANDARD.ordinal()) {
- int numPartitions = partitions_.size();
if (tbl_.getNumClusteringCols() == 0) numPartitions = 1;
output.append(String.format("%spartitions=%s/%s files=%s size=%s", detailPrefix,
numPartitions, table.getPartitions().size() - 1, totalFiles_,
@@ -586,6 +625,12 @@ public class HdfsScanNode extends ScanNode {
if (detailLevel.ordinal() >= TExplainLevel.EXTENDED.ordinal()) {
output.append(getStatsExplainString(detailPrefix, detailLevel));
output.append("\n");
+ if (numScanRangesNoDiskIds_ > 0) {
+ output.append(String.format("%smissing disk ids: " +
+ "partitions=%s/%s files=%s/%s scan ranges %s/%s\n", detailPrefix,
+ numPartitionsNoDiskIds_, numPartitions, numFilesNoDiskIds_,
+ totalFiles_, numScanRangesNoDiskIds_, scanRanges_.size()));
+ }
}
return output.toString();
}
@@ -663,4 +708,6 @@ public class HdfsScanNode extends ScanNode {
@Override
public boolean hasCorruptTableStats() { return hasCorruptTableStats_; }
+
+ public boolean hasMissingDiskIds() { return numScanRangesNoDiskIds_ > 0; }
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fcc2d817/fe/src/main/java/org/apache/impala/planner/Planner.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/Planner.java b/fe/src/main/java/org/apache/impala/planner/Planner.java
index b47a7b3..cc8b39b 100644
--- a/fe/src/main/java/org/apache/impala/planner/Planner.java
+++ b/fe/src/main/java/org/apache/impala/planner/Planner.java
@@ -304,6 +304,16 @@ public class Planner {
hasHeader = true;
}
+ if (request.query_ctx.isSetTables_missing_diskids()) {
+ List<String> tableNames = Lists.newArrayList();
+ for (TTableName tableName: request.query_ctx.getTables_missing_diskids()) {
+ tableNames.add(tableName.db_name + "." + tableName.table_name);
+ }
+ str.append("WARNING: The following tables have scan ranges with missing " +
+ "disk id information.\n" + Joiner.on(", ").join(tableNames) + "\n");
+ hasHeader = true;
+ }
+
if (request.query_ctx.isDisable_spilling()) {
str.append("WARNING: Spilling is disabled for this query as a safety guard.\n" +
"Reason: Query option disable_unsafe_spills is set, at least one table\n" +
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fcc2d817/fe/src/main/java/org/apache/impala/service/Frontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index 1030111..a67ad0d 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -86,6 +86,7 @@ import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.NotImplementedException;
+import org.apache.impala.planner.HdfsScanNode;
import org.apache.impala.planner.PlanFragment;
import org.apache.impala.planner.Planner;
import org.apache.impala.planner.ScanNode;
@@ -947,14 +948,17 @@ public class Frontend {
LOG.trace("get scan range locations");
Set<TTableName> tablesMissingStats = Sets.newTreeSet();
Set<TTableName> tablesWithCorruptStats = Sets.newTreeSet();
+ Set<TTableName> tablesWithMissingDiskIds = Sets.newTreeSet();
for (ScanNode scanNode: scanNodes) {
result.putToPer_node_scan_ranges(
scanNode.getId().asInt(), scanNode.getScanRangeLocations());
- if (scanNode.isTableMissingStats()) {
- tablesMissingStats.add(scanNode.getTupleDesc().getTableName().toThrift());
- }
- if (scanNode.hasCorruptTableStats()) {
- tablesWithCorruptStats.add(scanNode.getTupleDesc().getTableName().toThrift());
+
+ TTableName tableName = scanNode.getTupleDesc().getTableName().toThrift();
+ if (scanNode.isTableMissingStats()) tablesMissingStats.add(tableName);
+ if (scanNode.hasCorruptTableStats()) tablesWithCorruptStats.add(tableName);
+ if (scanNode instanceof HdfsScanNode &&
+ ((HdfsScanNode) scanNode).hasMissingDiskIds()) {
+ tablesWithMissingDiskIds.add(tableName);
}
}
@@ -964,6 +968,9 @@ public class Frontend {
for (TTableName tableName: tablesWithCorruptStats) {
queryCtx.addToTables_with_corrupt_stats(tableName);
}
+ for (TTableName tableName: tablesWithMissingDiskIds) {
+ queryCtx.addToTables_missing_diskids(tableName);
+ }
// Compute resource requirements after scan range locations because the cost
// estimates of scan nodes rely on them.
[2/5] incubator-impala git commit: IMPALA-4738: STDDEV_SAMP should
return NULL for single record input
Posted by jr...@apache.org.
IMPALA-4738: STDDEV_SAMP should return NULL for single record input
In calculating the STDDEV_SAMP/VARIANCE of N rows a divion by
N-1 rows is involved. Hence STDDEV_SAMP/VARIANCE for a single
row involves a division by 0. This change returns a NULL instead
of a 0 when calculating STDDEV_SAMP/VARIANCE for a single row.
STDDEV_POP/VARIANCE_POP for single row will still return a 0 since
this does not involve a division by 0. This matches the postgres
behavior.
Change-Id: Ide8af752cd8a2e554a2cd5a1ec948967a80de1fe
Reviewed-on: http://gerrit.cloudera.org:8080/5800
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/c671ee1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/c671ee1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/c671ee1e
Branch: refs/heads/master
Commit: c671ee1ece5fde1a5db40621cb27480dff74bc49
Parents: 3671175
Author: aphadke <ap...@cloudera.com>
Authored: Wed Jan 25 14:47:30 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Feb 7 04:19:06 2017 +0000
----------------------------------------------------------------------
be/src/exprs/aggregate-functions-ir.cc | 4 ++--
.../functional-query/queries/QueryTest/aggregation.test | 11 ++++++-----
2 files changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c671ee1e/be/src/exprs/aggregate-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/aggregate-functions-ir.cc b/be/src/exprs/aggregate-functions-ir.cc
index 63f06bf..8505d43 100644
--- a/be/src/exprs/aggregate-functions-ir.cc
+++ b/be/src/exprs/aggregate-functions-ir.cc
@@ -1331,7 +1331,7 @@ DoubleVal AggregateFunctions::KnuthVarFinalize(
FunctionContext* ctx, const StringVal& state_sv) {
DCHECK(!state_sv.is_null);
KnuthVarianceState* state = reinterpret_cast<KnuthVarianceState*>(state_sv.ptr);
- if (state->count == 0) return DoubleVal::null();
+ if (state->count == 0 || state->count == 1) return DoubleVal::null();
double variance = ComputeKnuthVariance(*state, false);
return DoubleVal(variance);
}
@@ -1350,7 +1350,7 @@ DoubleVal AggregateFunctions::KnuthStddevFinalize(FunctionContext* ctx,
DCHECK(!state_sv.is_null);
DCHECK_EQ(state_sv.len, sizeof(KnuthVarianceState));
KnuthVarianceState* state = reinterpret_cast<KnuthVarianceState*>(state_sv.ptr);
- if (state->count == 0) return DoubleVal::null();
+ if (state->count == 0 || state->count == 1) return DoubleVal::null();
return sqrt(ComputeKnuthVariance(*state, false));
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c671ee1e/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/aggregation.test b/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
index 1b4d464..675c75f 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
@@ -22,14 +22,15 @@ NULL,NULL,NULL,NULL
double, double, double, double
====
---- QUERY
-# exactly 1 tuple processed (variance & stddev are 0)
-SELECT variance(tinyint_col), stddev(smallint_col), variance_pop(int_col),
-stddev_pop(bigint_col)
+# exactly 1 tuple processed (variance_pop & stddev_pop are 0, stddev and variance
+# are NULL)
+SELECT variance(tinyint_col), variance_samp(smallint_col), variance_pop(int_col),
+stddev(smallint_col), stddev_samp(smallint_col), stddev_pop(bigint_col)
from alltypesagg WHERE id = 1006
---- RESULTS
-0,0,0,0
+NULL,NULL,0,NULL,NULL,0
---- TYPES
-double, double, double, double
+double, double, double, double, double, double
====
---- QUERY
# Includes one row which is null, and test the aliases for variance() as well