You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/08/31 05:32:20 UTC
[1/7] incubator-impala git commit: IMPALA-5857: avoid invalid free of
hedged read metrics
Repository: incubator-impala
Updated Branches:
refs/heads/release-2.10.0 [created] 23d79462d
IMPALA-5857: avoid invalid free of hedged read metrics
The libHdfs API documents that the output parameter is unchanged on
error, therefore we do not need to attempt to free it on error.
Testing:
The bug only reproduced under stress. I don't know how to trigger this
error path yet.
Change-Id: I93baf3b672429c0283d7f031ff302aca31e05be4
Reviewed-on: http://gerrit.cloudera.org:8080/7885
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/2912a0f9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/2912a0f9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/2912a0f9
Branch: refs/heads/release-2.10.0
Commit: 2912a0f9d9b32caf586b9383c7e027af3fe4c5c4
Parents: ebe8ddd
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Tue Aug 29 15:29:50 2017 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Wed Aug 30 14:54:49 2017 -0700
----------------------------------------------------------------------
be/src/runtime/disk-io-mgr-scan-range.cc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/2912a0f9/be/src/runtime/disk-io-mgr-scan-range.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/disk-io-mgr-scan-range.cc b/be/src/runtime/disk-io-mgr-scan-range.cc
index f9f96d0..df74fcc 100644
--- a/be/src/runtime/disk-io-mgr-scan-range.cc
+++ b/be/src/runtime/disk-io-mgr-scan-range.cc
@@ -350,14 +350,14 @@ void DiskIoMgr::ScanRange::Close() {
// Update Hedged Read Metrics.
// We call it only if the --use_hdfs_pread flag is set, to avoid having the
// libhdfs client malloc and free a hdfsHedgedReadMetrics object unnecessarily
- // otherwise.
+ // otherwise. 'hedged_metrics' is only set upon success.
struct hdfsHedgedReadMetrics* hedged_metrics;
int success = hdfsGetHedgedReadMetrics(fs_, &hedged_metrics);
if (success == 0) {
ImpaladMetrics::HEDGED_READ_OPS->set_value(hedged_metrics->hedgedReadOps);
ImpaladMetrics::HEDGED_READ_OPS_WIN->set_value(hedged_metrics->hedgedReadOpsWin);
+ hdfsFreeHedgedReadMetrics(hedged_metrics);
}
- hdfsFreeHedgedReadMetrics(hedged_metrics);
}
if (num_remote_bytes_ > 0) {
[5/7] incubator-impala git commit: IMPALA-5840: Don't write
page-level statistics in Parquet files.
Posted by ta...@apache.org.
IMPALA-5840: Don't write page-level statistics in Parquet files.
Page level statistics in Parquet files are expected to be deprecated in
favor of page indexes (PARQUET-922). This change disables writing
statistics to pages. Impala is currently the only project writing them.
Neither Impala nor other projects make use of these right now and by not
writing them anymore we prevent others from depending on soon-to-be
deprecated fields.
Change-Id: I1b05131320370171d76e93a46b04880a7f9b6d84
Reviewed-on: http://gerrit.cloudera.org:8080/7817
Reviewed-by: Lars Volker <lv...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/1faf89f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/1faf89f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/1faf89f0
Branch: refs/heads/release-2.10.0
Commit: 1faf89f047e7d78c3a1f3b518269a3ae21a4ddea
Parents: 73cb9b8
Author: Lars Volker <lv...@cloudera.com>
Authored: Thu Aug 24 14:59:22 2017 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Wed Aug 30 14:54:49 2017 -0700
----------------------------------------------------------------------
be/src/exec/hdfs-parquet-table-writer.cc | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1faf89f0/be/src/exec/hdfs-parquet-table-writer.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-table-writer.cc b/be/src/exec/hdfs-parquet-table-writer.cc
index 237dd83..4bbadb4 100644
--- a/be/src/exec/hdfs-parquet-table-writer.cc
+++ b/be/src/exec/hdfs-parquet-table-writer.cc
@@ -385,7 +385,8 @@ class HdfsParquetTableWriter::ColumnWriter :
// Temporary string value to hold CHAR(N)
StringValue temp_;
- // Tracks statistics per page.
+ // Tracks statistics per page. These are not written out currently but are merged into
+ // the row group stats. TODO(IMPALA-5841): Write these to the page index.
scoped_ptr<ColumnStats<T>> page_stats_;
// Tracks statistics per row group. This gets reset when starting a new row group.
@@ -453,7 +454,8 @@ class HdfsParquetTableWriter::BoolColumnWriter :
// Used to encode bools as single bit values. This is reused across pages.
BitWriter* bool_values_;
- // Tracks statistics per page.
+ // Tracks statistics per page. These are not written out currently but are merged into
+ // the row group stats. TODO(IMPALA-5841): Write these to the page index.
ColumnStats<bool> page_stats_;
// Tracks statistics per row group. This gets reset when starting a new file.
@@ -695,15 +697,9 @@ Status HdfsParquetTableWriter::BaseColumnWriter::FinalizeCurrentPage() {
max_compressed_size - header.compressed_page_size);
}
- // Build page statistics and add them to the header.
- DCHECK(page_stats_base_ != nullptr);
- if (page_stats_base_->BytesNeeded() <= MAX_COLUMN_STATS_SIZE) {
- page_stats_base_->EncodeToThrift(&header.data_page_header.statistics);
- header.data_page_header.__isset.statistics = true;
- }
-
// Update row group statistics from page statistics.
DCHECK(row_group_stats_base_ != nullptr);
+ DCHECK(page_stats_base_ != nullptr);
row_group_stats_base_->Merge(*page_stats_base_);
// Add the size of the data page header
[6/7] incubator-impala git commit: IMPALA-5850: Cast sender partition
exprs under unions.
Posted by ta...@apache.org.
IMPALA-5850: Cast sender partition exprs under unions.
For a series of partitioned joins within the same fragment we must
cast the sender partition exprs of exchanges to compatible types.
Otherwise, the hashes generated for identical partition values may
differ among senders leading to wrong results.
The bug was that this casting process was only performed for
fragments that are hash-partitioned. However, a union produces a
fragment with RANDOM partition, but the union could still contain
partitioned joins whose senders need to be cast appropriately. The
fix is to add casts regardless of the fragment's data partition.
Testing:
- Core/hdfs run passed
- Added a new regresion test
Change-Id: I0aa801bcad8c2324d848349c7967d949224404e0
Reviewed-on: http://gerrit.cloudera.org:8080/7884
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/a58394be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/a58394be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/a58394be
Branch: refs/heads/release-2.10.0
Commit: a58394be7c7998a5dfea53d8a3dbf8beb3370a48
Parents: 2912a0f
Author: Alex Behm <al...@cloudera.com>
Authored: Mon Aug 28 19:01:39 2017 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Wed Aug 30 14:54:50 2017 -0700
----------------------------------------------------------------------
.../org/apache/impala/planner/PlanFragment.java | 78 +++++++++++---------
.../queries/QueryTest/joins.test | 41 ++++++++++
2 files changed, 83 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a58394be/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/PlanFragment.java b/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
index ab72863..5ac9b2e 100644
--- a/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
+++ b/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
@@ -25,9 +25,9 @@ import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.TupleId;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.InternalException;
-import org.apache.impala.common.NotImplementedException;
import org.apache.impala.common.PrintUtils;
import org.apache.impala.common.TreeNode;
+import org.apache.impala.planner.JoinNode.DistributionMode;
import org.apache.impala.planner.PlanNode.ExecPhaseResourceProfiles;
import org.apache.impala.thrift.TExplainLevel;
import org.apache.impala.thrift.TPartitionType;
@@ -38,7 +38,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
-import com.google.common.base.Predicates;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -160,13 +159,12 @@ public class PlanFragment extends TreeNode<PlanFragment> {
/**
* Do any final work to set up the ExchangeNodes and DataStreamSinks for this fragment.
- * If this fragment is hash partitioned, ensures that the corresponding partition
- * exprs of all hash-partitioning senders are cast to identical types.
+ * If this fragment has partitioned joins, ensures that the corresponding partition
+ * exprs of all hash-partitioning senders are cast to appropriate types.
* Otherwise, the hashes generated for identical partition values may differ
* among senders if the partition-expr types are not identical.
*/
- public void finalizeExchanges(Analyzer analyzer)
- throws InternalException, NotImplementedException {
+ public void finalizeExchanges(Analyzer analyzer) throws InternalException {
if (destNode_ != null) {
Preconditions.checkState(sink_ == null);
// we're streaming to an exchange node
@@ -175,38 +173,46 @@ public class PlanFragment extends TreeNode<PlanFragment> {
sink_ = streamSink;
}
- if (!dataPartition_.isHashPartitioned()) return;
-
- // This fragment is hash partitioned. Gather all exchange nodes and ensure
- // that all hash-partitioning senders hash on exprs-values of the same type.
- List<ExchangeNode> exchNodes = Lists.newArrayList();
- planRoot_.collect(Predicates.instanceOf(ExchangeNode.class), exchNodes);
+ // Must be called regardless of this fragment's data partition. This fragment might
+ // be RANDOM partitioned due to a union. The union could still have partitioned joins
+ // in its child subtrees for which casts on the exchange senders are needed.
+ castPartitionedJoinExchanges(planRoot_, analyzer);
+ }
- // Contains partition-expr lists of all hash-partitioning sender fragments.
- List<List<Expr>> senderPartitionExprs = Lists.newArrayList();
- for (ExchangeNode exchNode: exchNodes) {
- Preconditions.checkState(!exchNode.getChildren().isEmpty());
- PlanFragment senderFragment = exchNode.getChild(0).getFragment();
- Preconditions.checkNotNull(senderFragment);
- if (!senderFragment.getOutputPartition().isHashPartitioned()) continue;
- List<Expr> partExprs = senderFragment.getOutputPartition().getPartitionExprs();
- // All hash-partitioning senders must have compatible partition exprs, otherwise
- // this fragment's data partition must not be hash partitioned.
- Preconditions.checkState(
- partExprs.size() == dataPartition_.getPartitionExprs().size());
- senderPartitionExprs.add(partExprs);
- }
+ /**
+ * Recursively traverses the plan tree rooted at 'node' and casts the partition exprs
+ * of all senders feeding into a series of partitioned joins to compatible types.
+ */
+ private void castPartitionedJoinExchanges(PlanNode node, Analyzer analyzer) {
+ if (node instanceof HashJoinNode
+ && ((JoinNode) node).getDistributionMode() == DistributionMode.PARTITIONED) {
+ // Contains all exchange nodes in this fragment below the current join node.
+ List<ExchangeNode> exchNodes = Lists.newArrayList();
+ node.collect(ExchangeNode.class, exchNodes);
+
+ // Contains partition-expr lists of all hash-partitioning sender fragments.
+ List<List<Expr>> senderPartitionExprs = Lists.newArrayList();
+ for (ExchangeNode exchNode: exchNodes) {
+ Preconditions.checkState(!exchNode.getChildren().isEmpty());
+ PlanFragment senderFragment = exchNode.getChild(0).getFragment();
+ Preconditions.checkNotNull(senderFragment);
+ if (!senderFragment.getOutputPartition().isHashPartitioned()) continue;
+ List<Expr> partExprs = senderFragment.getOutputPartition().getPartitionExprs();
+ senderPartitionExprs.add(partExprs);
+ }
- // Cast all corresponding hash partition exprs of all hash-partitioning senders
- // to their compatible types. Also cast the data partition's exprs for consistency,
- // although not strictly necessary. They should already be type identical to the
- // exprs of one of the senders and they are not directly used for hashing in the BE.
- senderPartitionExprs.add(dataPartition_.getPartitionExprs());
- try {
- analyzer.castToUnionCompatibleTypes(senderPartitionExprs);
- } catch (AnalysisException e) {
- // Should never happen. Analysis should have ensured type compatibility already.
- throw new IllegalStateException(e);
+ // Cast partition exprs of all hash-partitioning senders to their compatible types.
+ try {
+ analyzer.castToUnionCompatibleTypes(senderPartitionExprs);
+ } catch (AnalysisException e) {
+ // Should never happen. Analysis should have ensured type compatibility already.
+ throw new IllegalStateException(e);
+ }
+ } else {
+ // Recursively traverse plan nodes in this fragment.
+ for (PlanNode child: node.getChildren()) {
+ if (child.getFragment() == this) castPartitionedJoinExchanges(child, analyzer);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a58394be/testdata/workloads/functional-query/queries/QueryTest/joins.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/joins.test b/testdata/workloads/functional-query/queries/QueryTest/joins.test
index db915df..ab2a531 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/joins.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/joins.test
@@ -730,3 +730,44 @@ t1.timestamp_col = cast(t2.d4 as TIMESTAMP);
---- TYPES
BIGINT
====
+---- QUERY
+# IMPALA-5850: Tests that the sender partition exprs of a series of partitioned hash
+# joins are cast to compatible types even when the containing fragment is RANDOM
+# partitioned due a union. This test borrows ideas from existing tests for IMPALA-1123.
+# The test has a union with three branches where each branch contains independent and
+# incompatible hash-partitioning exchanges. Two of the union branches contain
+# partitioned joins which require the sender partition exprs to be cast to produce
+# correct results.
+# Breakdown of result count:
+# The first union branch is expected to produce 2 rows
+# The second union branch is expected to produce 11200 rows (see IMPALA-1123 test)
+# The third union branch is expected to produce 10 rows
+select count(*) from (
+ select distinct tinyint_col, smallint_col, int_col
+ from functional.alltypestiny
+ union all
+ select /* +straight_join */ b.id, c.tinyint_col, null
+ from functional.alltypessmall a
+ inner join /* +shuffle */
+ (select /* +straight_join */ t2.id, t2.smallint_col
+ from functional.alltypessmall t1
+ inner join /* +shuffle */ functional.alltypessmall t2
+ on t1.tinyint_col = t2.smallint_col) b
+ on a.int_col = b.smallint_col
+ inner join /* +shuffle */
+ (select distinct tinyint_col
+ from functional.alltypessmall) c
+ on a.int_col = c.tinyint_col
+ union all
+ select /* +straight_join */ tinyint_col, bigint_col, null from
+ (select distinct tinyint_col, bigint_col div 10 as bigint_col
+ from functional.alltypessmall) a
+ inner join /* +shuffle */
+ (select distinct int_col, smallint_col
+ from functional.alltypessmall) b
+ on a.tinyint_col = b.int_col and a.bigint_col = b.smallint_col) v
+---- RESULTS
+11212
+---- TYPES
+BIGINT
+====
[4/7] incubator-impala git commit: IMPALA-5830:
SET_DENY_RESERVATION_PROBABILITY test
Posted by ta...@apache.org.
IMPALA-5830: SET_DENY_RESERVATION_PROBABILITY test
Add a targeted test that confirms that setting the query option will
force spilling.
Testing:
Ran test_spilling locally.
Change-Id: Ida6b55b2dee0779b1739af5d75943518ec40d6ce
Reviewed-on: http://gerrit.cloudera.org:8080/7809
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/ebe8ddd4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/ebe8ddd4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/ebe8ddd4
Branch: refs/heads/release-2.10.0
Commit: ebe8ddd451b3f14d3f778339978a76bcd14b2589
Parents: 1faf89f
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Thu Aug 24 13:26:46 2017 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Wed Aug 30 14:54:49 2017 -0700
----------------------------------------------------------------------
.../QueryTest/disable-unsafe-spills.test | 11 ----
.../QueryTest/spilling-query-options.test | 54 ++++++++++++++++++++
tests/common/impala_test_suite.py | 2 +-
tests/query_test/test_spilling.py | 41 +++++++++++----
4 files changed, 85 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ebe8ddd4/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test b/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
deleted file mode 100644
index a9b8f35..0000000
--- a/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
+++ /dev/null
@@ -1,11 +0,0 @@
-====
----- QUERY
-# tpch_avro does not have stats computed, so if we set disable_unsafe_spills we should
-# not spill to disk.
-set disable_unsafe_spills=true;
-set buffer_pool_limit=40m;
-select distinct *
-from tpch_avro.orders
----- CATCH
-Could not free memory by spilling to disk: spilling was disabled by planner. Re-enable spilling by setting the query option DISABLE_UNSAFE_SPILLS=false
-====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ebe8ddd4/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test
new file mode 100644
index 0000000..5b9a641
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test
@@ -0,0 +1,54 @@
+====
+---- QUERY
+# tpch_avro does not have stats computed, so if we set disable_unsafe_spills we should
+# not spill to disk.
+set disable_unsafe_spills=true;
+set buffer_pool_limit=40m;
+select distinct *
+from tpch_avro.orders
+---- CATCH
+Could not free memory by spilling to disk: spilling was disabled by planner. Re-enable spilling by setting the query option DISABLE_UNSAFE_SPILLS=false
+====
+---- QUERY
+# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on PREPARE.
+set debug_action="-1:PREPARE:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select count(*) from (
+ select distinct o_orderdate, o_custkey, o_comment
+ from tpch_parquet.orders where o_orderkey < 500000) v;
+---- TYPES
+BIGINT
+---- RESULTS
+124999
+---- RUNTIME_PROFILE
+row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
+row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on OPEN.
+set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select count(*) from (
+ select distinct o_orderdate, o_custkey, o_comment
+ from tpch_parquet.orders where o_orderkey < 500000) v;
+---- TYPES
+BIGINT
+---- RESULTS
+124999
+---- RUNTIME_PROFILE
+row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
+row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on GETNEXT.
+# This won't affect the merge aggregation, which accumulates its memory in Open(), but
+# will affect the streaming aggregation.
+set debug_action="-1:GETNEXT:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select count(*) from (
+ select distinct o_orderdate, o_custkey, o_comment
+ from tpch_parquet.orders where o_orderkey < 500000) v;
+---- TYPES
+BIGINT
+---- RESULTS
+124999
+---- RUNTIME_PROFILE
+row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ebe8ddd4/tests/common/impala_test_suite.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index d5841b2..1b7d043 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -210,7 +210,7 @@ class ImpalaTestSuite(BaseTestSuite):
if not query_option in self.default_query_options:
continue
default_val = self.default_query_options[query_option]
- query_str = 'SET '+ query_option + '=' + default_val + ';'
+ query_str = 'SET '+ query_option + '="' + default_val + '"'
try:
impalad_client.execute(query_str)
except Exception as e:
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ebe8ddd4/tests/query_test/test_spilling.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py
index c818b65..0b36429 100644
--- a/tests/query_test/test_spilling.py
+++ b/tests/query_test/test_spilling.py
@@ -30,14 +30,14 @@ DEBUG_ACTION_DIMS = [None,
@pytest.mark.xfail(pytest.config.option.testing_remote_cluster,
reason='Queries may not spill on larger clusters')
-class TestSpilling(ImpalaTestSuite):
+class TestSpillingDebugActionDimensions(ImpalaTestSuite):
@classmethod
def get_workload(self):
return 'functional-query'
@classmethod
def add_test_dimensions(cls):
- super(TestSpilling, cls).add_test_dimensions()
+ super(TestSpillingDebugActionDimensions, cls).add_test_dimensions()
cls.ImpalaTestMatrix.clear_constraints()
cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch'))
# Tests are calibrated so that they can execute and spill with this page size.
@@ -60,19 +60,38 @@ class TestSpilling(ImpalaTestSuite):
"""Test spilling null-aware anti-joins"""
self.run_test_case('QueryTest/spilling-naaj', vector)
+ def test_spilling_sorts_exhaustive(self, vector):
+ if self.exploration_strategy() != 'exhaustive':
+ pytest.skip("only run large sorts on exhaustive")
+ self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector)
+
+
+@pytest.mark.xfail(pytest.config.option.testing_remote_cluster,
+ reason='Queries may not spill on larger clusters')
+class TestSpillingNoDebugActionDimensions(ImpalaTestSuite):
+ """Spilling tests to which we don't want to apply the debug_action dimension."""
+ @classmethod
+ def get_workload(self):
+ return 'functional-query'
+
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestSpillingNoDebugActionDimensions, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.clear_constraints()
+ cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch'))
+ # Tests are calibrated so that they can execute and spill with this page size.
+ cls.ImpalaTestMatrix.add_dimension(
+ create_exec_option_dimension_from_dict({'default_spillable_buffer_size' : ['256k']}))
+
def test_spilling_naaj_no_deny_reservation(self, vector):
"""
Null-aware anti-join tests that depend on getting more than the minimum reservation
and therefore will not reliably pass with the deny reservation debug action enabled.
"""
- if vector.get_value('exec_option')['debug_action'] is None:
- self.run_test_case('QueryTest/spilling-naaj-no-deny-reservation', vector)
+ self.run_test_case('QueryTest/spilling-naaj-no-deny-reservation', vector)
- def test_spilling_sorts_exhaustive(self, vector):
- if self.exploration_strategy() != 'exhaustive':
- pytest.skip("only run large sorts on exhaustive")
- self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector)
+ def test_spilling_query_options(self, vector):
+ """Test that spilling-related query options work end-to-end. These tests rely on
+ setting debug_action to alternative values via query options."""
+ self.run_test_case('QueryTest/spilling-query-options', vector)
- def test_disable_unsafe_spills(self, vector):
- """Test that the disable_unsafe_spills query options works end-to-end."""
- self.run_test_case('QueryTest/disable-unsafe-spills', vector)
[2/7] incubator-impala git commit: IMPALA-5852: improve
MINIMUM_RESERVATION_UNAVAILABLE error
Posted by ta...@apache.org.
IMPALA-5852: improve MINIMUM_RESERVATION_UNAVAILABLE error
Augment the error message to mention that oversubscription is likely the
problem and hint at solutions.
Change-Id: I8e367e1b0cb08e11fdd0546880df23b785e3b7c9
Reviewed-on: http://gerrit.cloudera.org:8080/7861
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/73cb9b8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/73cb9b8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/73cb9b8b
Branch: refs/heads/release-2.10.0
Commit: 73cb9b8b0f6020fb90acf4fa12a00753a3120058
Parents: 99fe9b3
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Mon Aug 28 15:03:38 2017 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Wed Aug 30 14:54:49 2017 -0700
----------------------------------------------------------------------
common/thrift/generate_error_codes.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/73cb9b8b/common/thrift/generate_error_codes.py
----------------------------------------------------------------------
diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py
index 1d3b7c6..393daba 100755
--- a/common/thrift/generate_error_codes.py
+++ b/common/thrift/generate_error_codes.py
@@ -323,8 +323,9 @@ error_codes = (
"Failed to verify generated IR function $0, see log for more details."),
("MINIMUM_RESERVATION_UNAVAILABLE", 106, "Failed to get minimum memory reservation of "
- "$0 on daemon $1:$2 for query $3 because it would exceed an applicable query, "
- "request pool or process memory limit. Memory usage:\\n$4"),
+ "$0 on daemon $1:$2 for query $3 because it would exceed an applicable memory "
+ "limit. Memory is likely oversubscribed. Reducing query concurrency or configuring "
+ "admission control may help avoid this error. Memory usage:\\n$4"),
("ADMISSION_REJECTED", 107, "Rejected query from pool $0: $1"),
[7/7] incubator-impala git commit: IMPALA-5855: reserve enough memory
for preaggs
Posted by ta...@apache.org.
IMPALA-5855: reserve enough memory for preaggs
The calculation in the planner failed to account for the behaviour of
Suballocator, which needs to obtain at least one buffer to allocate any
memory.
Testing:
Added a regression test that caused a crash before the fix.
Updated planner tests.
Was able to run local stress test binary search to completion (it
previously crashed).
Change-Id: I870fbe2f1da01c6123d3716a1198376f9a454c3b
Reviewed-on: http://gerrit.cloudera.org:8080/7871
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/23d79462
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/23d79462
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/23d79462
Branch: refs/heads/release-2.10.0
Commit: 23d79462da5d0108709e8b1399c97606f4ebdf92
Parents: a58394b
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Tue Aug 29 08:29:44 2017 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Wed Aug 30 14:55:08 2017 -0700
----------------------------------------------------------------------
.../apache/impala/planner/AggregationNode.java | 10 ++--
.../queries/PlannerTest/max-row-size.test | 18 +++---
.../queries/PlannerTest/mt-dop-validation.test | 4 +-
.../PlannerTest/resource-requirements.test | 60 ++++++++++----------
.../PlannerTest/spillable-buffer-sizing.test | 38 ++++++-------
.../admission-reject-min-reservation.test | 6 +-
.../queries/QueryTest/aggregation.test | 12 ++++
7 files changed, 81 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/AggregationNode.java b/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
index 2778a7f..c31f448 100644
--- a/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
@@ -323,10 +323,12 @@ public class AggregationNode extends PlanNode {
if (useStreamingPreagg_) {
// We can execute a streaming preagg without any buffers by passing through rows,
// but that is a very low performance mode of execution if the aggregation reduces
- // its input significantly. Instead reserve memory for one buffer and 64kb of hash
- // tables per partition. We don't need to reserve memory for large rows since they
- // can be passed through if needed.
- perInstanceMinReservation = (bufferSize + 64 * 1024) * PARTITION_FANOUT;
+ // its input significantly. Instead reserve memory for one buffer per partition
+ // and at least 64kb for hash tables per partition. We must reserve at least one
+ // full buffer for hash tables for the suballocator to subdivide. We don't need to
+ // reserve memory for large rows since they can be passed through if needed.
+ perInstanceMinReservation = bufferSize * PARTITION_FANOUT +
+ Math.max(64 * 1024 * PARTITION_FANOUT, bufferSize);
} else {
long minBuffers = PARTITION_FANOUT + 1 + (aggInfo_.needsSerialize() ? 1 : 0);
// Two of the buffers need to be buffers large enough to hold the maximum-sized
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test b/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
index 5a0a76c..d563444 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
@@ -150,7 +150,7 @@ from tpch_parquet.lineitem
group by 1, 2
having count(*) = 1
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=110.00MB
+Max Per-Host Resource Reservation: Memory=111.00MB
Per-Host Resource Estimates: Memory=251.12MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -176,11 +176,11 @@ Per-Host Resources: mem-estimate=46.00MB mem-reservation=46.00MB
| tuple-ids=2 row-size=33B cardinality=4690314
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
-Per-Host Resources: mem-estimate=85.12MB mem-reservation=64.00MB
+Per-Host Resources: mem-estimate=85.12MB mem-reservation=65.00MB
03:AGGREGATE [STREAMING]
| output: count(*)
| group by: l_orderkey, o_orderstatus
-| mem-estimate=54.12MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=54.12MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=2 row-size=33B cardinality=4690314
|
02:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -224,7 +224,7 @@ Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
select distinct *
from tpch_parquet.lineitem
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=79.00MB
+Max Per-Host Resource Reservation: Memory=80.00MB
Per-Host Resource Estimates: Memory=3.31GB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -248,10 +248,10 @@ Per-Host Resources: mem-estimate=1.62GB mem-reservation=46.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=1.69GB mem-reservation=33.00MB
+Per-Host Resources: mem-estimate=1.69GB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
-| mem-estimate=1.62GB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
@@ -268,7 +268,7 @@ select l_orderkey, l_partkey, group_concat(l_linestatus, ",")
from tpch_parquet.lineitem
group by 1, 2
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=81.00MB
+Max Per-Host Resource Reservation: Memory=82.00MB
Per-Host Resource Estimates: Memory=482.91MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -293,11 +293,11 @@ Per-Host Resources: mem-estimate=201.46MB mem-reservation=48.00MB
| tuple-ids=1 row-size=32B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=281.46MB mem-reservation=33.00MB
+Per-Host Resources: mem-estimate=281.46MB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| output: group_concat(l_linestatus, ',')
| group by: l_orderkey, l_partkey
-| mem-estimate=201.46MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=201.46MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=32B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test b/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
index 9f80785..dcebf07 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
@@ -95,11 +95,11 @@ Per-Host Resources: mem-estimate=384.00MB mem-reservation=102.00MB
| tuple-ids=1 row-size=16B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9
-Per-Host Resources: mem-estimate=432.00MB mem-reservation=99.00MB
+Per-Host Resources: mem-estimate=432.00MB mem-reservation=102.00MB
01:AGGREGATE [STREAMING]
| output: count(int_col)
| group by: bigint_col
-| mem-estimate=128.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=128.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=16B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
index 90035c1..fe3fd41 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
@@ -375,8 +375,8 @@ PLAN-ROOT SINK
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=41.50MB
-Per-Host Resource Estimates: Memory=123.00MB
+Max Per-Host Resource Reservation: Memory=42.50MB
+Per-Host Resource Estimates: Memory=124.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -400,11 +400,11 @@ Per-Host Resources: mem-estimate=10.00MB mem-reservation=8.50MB
| tuple-ids=1 row-size=16B cardinality=1563438
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=113.00MB mem-reservation=33.00MB
+Per-Host Resources: mem-estimate=114.00MB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: l_orderkey
-| mem-estimate=33.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=16B cardinality=1563438
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
@@ -415,8 +415,8 @@ Per-Host Resources: mem-estimate=113.00MB mem-reservation=33.00MB
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=75.50MB
-Per-Host Resource Estimates: Memory=246.00MB
+Max Per-Host Resource Reservation: Memory=77.50MB
+Per-Host Resource Estimates: Memory=248.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -440,11 +440,11 @@ Per-Host Resources: mem-estimate=20.00MB mem-reservation=9.50MB
| tuple-ids=1 row-size=16B cardinality=1563438
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=226.00MB mem-reservation=66.00MB
+Per-Host Resources: mem-estimate=228.00MB mem-reservation=68.00MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: l_orderkey
-| mem-estimate=33.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=16B cardinality=1563438
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
@@ -1548,7 +1548,7 @@ PLAN-ROOT SINK
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=78B cardinality=600122
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=71.75MB
+Max Per-Host Resource Reservation: Memory=72.75MB
Per-Host Resource Estimates: Memory=344.33MB
F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1643,10 +1643,10 @@ Per-Host Resources: mem-estimate=97.00MB mem-reservation=34.00MB
| tuple-ids=2 row-size=70B cardinality=575772
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
-Per-Host Resources: mem-estimate=47.33MB mem-reservation=37.75MB
+Per-Host Resources: mem-estimate=47.33MB mem-reservation=38.75MB
04:AGGREGATE [STREAMING]
| group by: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
-| mem-estimate=42.58MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=42.58MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=2 row-size=70B cardinality=575772
|
03:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -1688,7 +1688,7 @@ Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=78B cardinality=600122
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=139.75MB
+Max Per-Host Resource Reservation: Memory=141.75MB
Per-Host Resource Estimates: Memory=684.91MB
F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1799,10 +1799,10 @@ Per-Host Resources: mem-estimate=194.00MB mem-reservation=68.00MB
| tuple-ids=2 row-size=70B cardinality=575772
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
-Per-Host Resources: mem-estimate=90.91MB mem-reservation=71.75MB
+Per-Host Resources: mem-estimate=90.91MB mem-reservation=73.75MB
04:AGGREGATE [STREAMING]
| group by: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
-| mem-estimate=42.58MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=42.58MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=2 row-size=70B cardinality=575772
|
03:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -1968,7 +1968,7 @@ PLAN-ROOT SINK
mem-estimate=88.00MB mem-reservation=0B
tuple-ids=2 row-size=16B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=150.12MB
+Max Per-Host Resource Reservation: Memory=152.12MB
Per-Host Resource Estimates: Memory=511.55MB
F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2000,11 +2000,11 @@ Per-Host Resources: mem-estimate=60.41MB mem-reservation=34.00MB
| tuple-ids=6 row-size=100B cardinality=575772
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
-Per-Host Resources: mem-estimate=115.78MB mem-reservation=83.12MB
+Per-Host Resources: mem-estimate=115.78MB mem-reservation=84.12MB
08:AGGREGATE [STREAMING]
| output: sum(l_quantity)
| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
-| mem-estimate=60.40MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=60.40MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=6 row-size=100B cardinality=575772
|
07:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]
@@ -2025,11 +2025,11 @@ Per-Host Resources: mem-estimate=115.78MB mem-reservation=83.12MB
| | tuple-ids=4 row-size=24B cardinality=1563438
| |
| F04:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-| Per-Host Resources: mem-estimate=127.36MB mem-reservation=33.00MB
+| Per-Host Resources: mem-estimate=127.36MB mem-reservation=34.00MB
| 04:AGGREGATE [STREAMING]
| | output: sum(l_quantity)
| | group by: l_orderkey
-| | mem-estimate=39.36MB mem-reservation=33.00MB spill-buffer=2.00MB
+| | mem-estimate=39.36MB mem-reservation=34.00MB spill-buffer=2.00MB
| | tuple-ids=4 row-size=24B cardinality=1563438
| |
| 03:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -2098,7 +2098,7 @@ Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
mem-estimate=88.00MB mem-reservation=0B
tuple-ids=2 row-size=16B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=254.88MB
+Max Per-Host Resource Reservation: Memory=258.88MB
Per-Host Resource Estimates: Memory=967.22MB
F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2130,11 +2130,11 @@ Per-Host Resources: mem-estimate=120.82MB mem-reservation=68.00MB
| tuple-ids=6 row-size=100B cardinality=575772
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
-Per-Host Resources: mem-estimate=175.68MB mem-reservation=120.88MB
+Per-Host Resources: mem-estimate=175.68MB mem-reservation=122.88MB
08:AGGREGATE [STREAMING]
| output: sum(l_quantity)
| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
-| mem-estimate=60.40MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=60.40MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=6 row-size=100B cardinality=575772
|
07:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]
@@ -2163,11 +2163,11 @@ Per-Host Resources: mem-estimate=175.68MB mem-reservation=120.88MB
| | tuple-ids=4 row-size=24B cardinality=1563438
| |
| F04:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-| Per-Host Resources: mem-estimate=254.73MB mem-reservation=66.00MB
+| Per-Host Resources: mem-estimate=254.73MB mem-reservation=68.00MB
| 04:AGGREGATE [STREAMING]
| | output: sum(l_quantity)
| | group by: l_orderkey
-| | mem-estimate=39.36MB mem-reservation=33.00MB spill-buffer=2.00MB
+| | mem-estimate=39.36MB mem-reservation=34.00MB spill-buffer=2.00MB
| | tuple-ids=4 row-size=24B cardinality=1563438
| |
| 03:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -2452,7 +2452,7 @@ PLAN-ROOT SINK
mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=66B cardinality=150000
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=102.94MB
+Max Per-Host Resource Reservation: Memory=103.94MB
Per-Host Resource Estimates: Memory=473.94MB
WARNING: The following tables are missing relevant table and/or column statistics.
tpch_nested_parquet.customer
@@ -2478,10 +2478,10 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=34.00MB
| tuple-ids=6 row-size=58B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=345.94MB mem-reservation=68.94MB
+Per-Host Resources: mem-estimate=345.94MB mem-reservation=69.94MB
09:AGGREGATE [STREAMING]
| group by: c_name, o1.o_orderkey, o2.o_orderstatus
-| mem-estimate=128.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=128.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=6 row-size=58B cardinality=1500000
|
01:SUBPLAN
@@ -2531,7 +2531,7 @@ Per-Host Resources: mem-estimate=345.94MB mem-reservation=68.94MB
mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=66B cardinality=150000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=205.88MB
+Max Per-Host Resource Reservation: Memory=207.88MB
Per-Host Resource Estimates: Memory=947.88MB
WARNING: The following tables are missing relevant table and/or column statistics.
tpch_nested_parquet.customer
@@ -2557,10 +2557,10 @@ Per-Host Resources: mem-estimate=256.00MB mem-reservation=68.00MB
| tuple-ids=6 row-size=58B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=691.88MB mem-reservation=137.88MB
+Per-Host Resources: mem-estimate=691.88MB mem-reservation=139.88MB
09:AGGREGATE [STREAMING]
| group by: c_name, o1.o_orderkey, o2.o_orderstatus
-| mem-estimate=128.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=128.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=6 row-size=58B cardinality=1500000
|
01:SUBPLAN
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
index 115a8bd..719b07d 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
@@ -603,7 +603,7 @@ from tpch_parquet.lineitem
group by 1, 2
having count(*) = 1
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=84.00MB
+Max Per-Host Resource Reservation: Memory=85.00MB
Per-Host Resource Estimates: Memory=225.12MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -629,11 +629,11 @@ Per-Host Resources: mem-estimate=34.00MB mem-reservation=34.00MB
| tuple-ids=2 row-size=33B cardinality=4690314
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
-Per-Host Resources: mem-estimate=71.12MB mem-reservation=50.00MB
+Per-Host Resources: mem-estimate=71.12MB mem-reservation=51.00MB
03:AGGREGATE [STREAMING]
| output: count(*)
| group by: l_orderkey, o_orderstatus
-| mem-estimate=54.12MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=54.12MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=2 row-size=33B cardinality=4690314
|
02:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -672,8 +672,8 @@ Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=117.00MB
-Per-Host Resource Estimates: Memory=357.00MB
+Max Per-Host Resource Reservation: Memory=119.00MB
+Per-Host Resource Estimates: Memory=359.00MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -698,11 +698,11 @@ Per-Host Resources: mem-estimate=34.00MB mem-reservation=34.00MB
| tuple-ids=2 row-size=33B cardinality=4690314
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
-Per-Host Resources: mem-estimate=83.00MB mem-reservation=83.00MB
+Per-Host Resources: mem-estimate=85.00MB mem-reservation=85.00MB
03:AGGREGATE [STREAMING]
| output: count(*)
| group by: l_orderkey, o_orderstatus
-| mem-estimate=33.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=2 row-size=33B cardinality=4690314
|
02:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -753,7 +753,7 @@ Per-Host Resources: mem-estimate=160.00MB mem-reservation=0B
select distinct *
from tpch_parquet.lineitem
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=67.00MB
+Max Per-Host Resource Reservation: Memory=68.00MB
Per-Host Resource Estimates: Memory=3.31GB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -777,10 +777,10 @@ Per-Host Resources: mem-estimate=1.62GB mem-reservation=34.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=1.69GB mem-reservation=33.00MB
+Per-Host Resources: mem-estimate=1.69GB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
-| mem-estimate=1.62GB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
@@ -791,7 +791,7 @@ Per-Host Resources: mem-estimate=1.69GB mem-reservation=33.00MB
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=134.00MB
+Max Per-Host Resource Reservation: Memory=136.00MB
Per-Host Resource Estimates: Memory=6.62GB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -815,10 +815,10 @@ Per-Host Resources: mem-estimate=3.23GB mem-reservation=68.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=3.39GB mem-reservation=66.00MB
+Per-Host Resources: mem-estimate=3.39GB mem-reservation=68.00MB
01:AGGREGATE [STREAMING]
| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
-| mem-estimate=1.62GB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
@@ -834,7 +834,7 @@ select string_col, count(*)
from functional_parquet.alltypestiny
group by string_col
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=67.00MB
+Max Per-Host Resource Reservation: Memory=68.00MB
Per-Host Resource Estimates: Memory=272.00MB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypestiny
@@ -861,11 +861,11 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=34.00MB
| tuple-ids=1 row-size=24B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=144.00MB mem-reservation=33.00MB
+Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: string_col
-| mem-estimate=128.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=128.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=24B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypestiny, RANDOM]
@@ -876,7 +876,7 @@ Per-Host Resources: mem-estimate=144.00MB mem-reservation=33.00MB
mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=16B cardinality=unavailable
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=134.00MB
+Max Per-Host Resource Reservation: Memory=136.00MB
Per-Host Resource Estimates: Memory=544.00MB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypestiny
@@ -903,11 +903,11 @@ Per-Host Resources: mem-estimate=256.00MB mem-reservation=68.00MB
| tuple-ids=1 row-size=24B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=288.00MB mem-reservation=66.00MB
+Per-Host Resources: mem-estimate=288.00MB mem-reservation=68.00MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: string_col
-| mem-estimate=128.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=128.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=24B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypestiny, RANDOM]
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
index 940fd6e..a84ec34 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
@@ -5,7 +5,7 @@ select distinct * from functional_parquet.alltypesagg
---- CATCH
minimum memory reservation is greater than memory available to the
query for buffer reservations. Memory reservation needed given the
- current plan: 67.00 MB. Set mem_limit to at least 142.00 MB.
+ current plan: 68.00 MB. Set mem_limit to at least 143.00 MB.
====
---- QUERY
set mem_limit=150mb;
@@ -13,14 +13,14 @@ select distinct * from functional_parquet.alltypesagg
---- CATCH
minimum memory reservation needed is greater than pool max mem resources.
Pool max mem resources: 10.00 MB.
- Cluster-wide memory reservation needed: 201.00 MB
+ Cluster-wide memory reservation needed: 204.00 MB
====
---- QUERY
set buffer_pool_limit=10mb;
select distinct * from functional_parquet.alltypesagg
---- CATCH
minimum memory reservation is greater than memory available to the query
- for buffer reservations. Increase the buffer_pool_limit to 67.00 MB.
+ for buffer reservations. Increase the buffer_pool_limit to 68.00 MB.
====
---- QUERY
set mem_limit=1024;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/aggregation.test b/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
index de1c507..3cc2f20 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
@@ -1319,3 +1319,15 @@ select 1 from functional_parquet.alltypes having count(*) > 1
---- TYPES
tinyint
====
+---- QUERY
+# IMPALA-5855: pre-aggregation does not reserve enough memory with 2MB buffers.
+# The pre-aggregation in this query is estimated to consume enough memory for the planner
+# to use 2MB buffers.
+set debug_action="-1:PREPARE:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select count(*) from (
+ select distinct l_orderkey, l_comment from tpch_parquet.lineitem) v
+---- RESULTS
+6001198
+---- TYPES
+bigint
+====
[3/7] incubator-impala git commit: IMPALA-5838: Improve errors on AC
buffer mem rejection
Posted by ta...@apache.org.
IMPALA-5838: Improve errors on AC buffer mem rejection
The error message returned when a query is rejected due to
insufficient buffer memory is misleading. It recommended a
mem_limit which would be high enough, but changing the
mem_limit may result in changing the plan, which may result
in further changes to the buffer memory requirement.
In particular, this can happen when the planner compares the
expected hash table size to the mem_limit, and decides to
choose a partitioned join over a broadcast join.
While we might consider other code changes to improve this,
for now lets just be clear in the error message.
Testing:
* Adds tests that verify the expected behavior with the new
error message.
Change-Id: I3dc3517195508d86078a8a4b537ae7d2f52fbcb7
Reviewed-on: http://gerrit.cloudera.org:8080/7834
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/99fe9b3f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/99fe9b3f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/99fe9b3f
Branch: refs/heads/release-2.10.0
Commit: 99fe9b3fd602180d63cbfe73ac2c9171c31ae455
Parents: 2a7c8b9
Author: Matthew Jacobs <mj...@cloudera.com>
Authored: Fri Aug 25 13:41:05 2017 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Wed Aug 30 14:54:49 2017 -0700
----------------------------------------------------------------------
be/src/scheduling/admission-controller.cc | 27 ++++++-------
.../admission-reject-min-reservation.test | 42 ++++++++++++++++++++
.../custom_cluster/test_admission_controller.py | 23 +----------
3 files changed, 57 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/99fe9b3f/be/src/scheduling/admission-controller.cc
----------------------------------------------------------------------
diff --git a/be/src/scheduling/admission-controller.cc b/be/src/scheduling/admission-controller.cc
index 339935d..1aadf22 100644
--- a/be/src/scheduling/admission-controller.cc
+++ b/be/src/scheduling/admission-controller.cc
@@ -115,18 +115,19 @@ const string PROFILE_INFO_VAL_QUEUE_DETAIL = "waited $0 ms, reason: $1";
// Error status string details
const string REASON_MEM_LIMIT_TOO_LOW_FOR_RESERVATION =
"minimum memory reservation is greater than memory available to the query "
- "for buffer reservations. Mem available for buffer reservations based on mem_limit: "
- "$0, memory reservation needed: $1. Set mem_limit to at least $2. See the query "
- "profile for more information.";
+ "for buffer reservations. Memory reservation needed given the current plan: $0. Set "
+ "mem_limit to at least $1. Note that changing the mem_limit may also change the "
+ "plan. See the query profile for more information about the per-node memory "
+ "requirements.";
const string REASON_BUFFER_LIMIT_TOO_LOW_FOR_RESERVATION =
"minimum memory reservation is greater than memory available to the query "
- "for buffer reservations. Mem available for buffer reservations based on "
- "buffer_pool_limit: $0, memory reservation needed: $1. See the query profile for "
- "more information.";
+ "for buffer reservations. Increase the buffer_pool_limit to $0. See the query "
+ "profile for more information about the per-node memory requirements.";
const string REASON_MIN_RESERVATION_OVER_POOL_MEM =
- "minimum memory reservation needed is greater than pool max mem resources. pool "
- "max mem resources: $0, cluster-wide memory reservation needed: $1. See the query "
- "profile for more information.";
+ "minimum memory reservation needed is greater than pool max mem resources. Pool "
+ "max mem resources: $0. Cluster-wide memory reservation needed: $1. Increase the "
+ "pool max mem resources. See the query profile for more information about the "
+ "per-node memory requirements.";
const string REASON_DISABLED_MAX_MEM_RESOURCES =
"disabled by pool max mem resources set to 0";
const string REASON_DISABLED_REQUESTS_LIMIT = "disabled by requests limit set to 0";
@@ -415,10 +416,9 @@ bool AdmissionController::RejectImmediately(QuerySchedule* schedule,
// Checks related to the min buffer reservation against configured query memory limits:
if (schedule->query_options().__isset.buffer_pool_limit &&
schedule->query_options().buffer_pool_limit > 0) {
- const int64_t buffer_pool_limit = schedule->query_options().buffer_pool_limit;
- if (max_min_reservation_bytes > buffer_pool_limit) {
+ if (max_min_reservation_bytes > schedule->query_options().buffer_pool_limit) {
*rejection_reason = Substitute(REASON_BUFFER_LIMIT_TOO_LOW_FOR_RESERVATION,
- PrintBytes(buffer_pool_limit), PrintBytes(max_min_reservation_bytes));
+ PrintBytes(max_min_reservation_bytes));
return true;
}
} else if (schedule->query_options().__isset.mem_limit &&
@@ -430,8 +430,7 @@ bool AdmissionController::RejectImmediately(QuerySchedule* schedule,
const int64_t required_mem_limit =
ReservationUtil::GetMinMemLimitFromReservation(max_min_reservation_bytes);
*rejection_reason = Substitute(REASON_MEM_LIMIT_TOO_LOW_FOR_RESERVATION,
- PrintBytes(mem_limit), PrintBytes(max_min_reservation_bytes),
- PrintBytes(required_mem_limit));
+ PrintBytes(max_min_reservation_bytes), PrintBytes(required_mem_limit));
return true;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/99fe9b3f/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
new file mode 100644
index 0000000..940fd6e
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
@@ -0,0 +1,42 @@
+====
+---- QUERY
+set mem_limit=10mb;
+select distinct * from functional_parquet.alltypesagg
+---- CATCH
+minimum memory reservation is greater than memory available to the
+ query for buffer reservations. Memory reservation needed given the
+ current plan: 67.00 MB. Set mem_limit to at least 142.00 MB.
+====
+---- QUERY
+set mem_limit=150mb;
+select distinct * from functional_parquet.alltypesagg
+---- CATCH
+minimum memory reservation needed is greater than pool max mem resources.
+ Pool max mem resources: 10.00 MB.
+ Cluster-wide memory reservation needed: 201.00 MB
+====
+---- QUERY
+set buffer_pool_limit=10mb;
+select distinct * from functional_parquet.alltypesagg
+---- CATCH
+minimum memory reservation is greater than memory available to the query
+ for buffer reservations. Increase the buffer_pool_limit to 67.00 MB.
+====
+---- QUERY
+set mem_limit=1024;
+select count(*)
+from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
+---- CATCH
+minimum memory reservation is greater than memory available to the
+ query for buffer reservations. Memory reservation needed given the
+ current plan: 4.75 MB. Set mem_limit to at least 79.75 MB.
+====
+---- QUERY
+set mem_limit=80mb;
+select count(*)
+from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
+---- CATCH
+minimum memory reservation is greater than memory available to the
+ query for buffer reservations. Memory reservation needed given the
+ current plan: 17.00 MB. Set mem_limit to at least 92.00 MB.
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/99fe9b3f/tests/custom_cluster/test_admission_controller.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index cf072b9..de97e7c 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -321,7 +321,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
impalad_args=impalad_admission_ctrl_flags(max_requests=1, max_queued=1,
pool_max_mem=10 * 1024 * 1024, proc_mem_limit=1024 * 1024 * 1024),
statestored_args=_STATESTORED_ARGS)
- def test_reject_min_reservation(self):
+ def test_reject_min_reservation(self, vector):
"""Test that the query will be rejected by admission control if:
a) the largest per-backend min buffer reservation is larger than the query mem
limit
@@ -330,26 +330,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
c) the cluster-wide min-buffer reservation size is larger than the pool memory
resources.
"""
- query = "select distinct * from functional_parquet.alltypesagg"
- opts = {'mem_limit': '10MB', 'num_nodes': '1'}
- ex = self.execute_query_expect_failure(self.client, query, opts)
- assert ("minimum memory reservation is greater than memory available to the query "
- "for buffer reservations. Mem available for buffer reservations based on "
- "mem_limit: 10.00 MB, memory reservation needed: 34.00 MB. "
- "Set mem_limit to at least 109.00 MB.") in str(ex)
-
- query = "select distinct * from functional_parquet.alltypesagg"
- opts = {'buffer_pool_limit': '10MB', 'num_nodes': '1'}
- ex = self.execute_query_expect_failure(self.client, query, opts)
- assert ("minimum memory reservation is greater than memory available to the query "
- "for buffer reservations. Mem available for buffer reservations based on "
- "buffer_pool_limit: 10.00 MB, memory reservation needed: 34.00 MB.") in str(ex)
-
- opts = {'mem_limit': '150MB', 'num_nodes': '1'}
- ex = self.execute_query_expect_failure(self.client, query, opts)
- assert ("minimum memory reservation needed is greater than pool max mem resources. "
- "pool max mem resources: 10.00 MB, cluster-wide memory reservation needed: "
- "34.00 MB") in str(ex)
+ self.run_test_case('QueryTest/admission-reject-min-reservation', vector)
# Process mem_limit used in test_mem_limit_upper_bound
PROC_MEM_TEST_LIMIT = 1024 * 1024 * 1024