You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/08/31 05:32:26 UTC
[7/7] incubator-impala git commit: IMPALA-5855: reserve enough memory
for preaggs
IMPALA-5855: reserve enough memory for preaggs
The calculation in the planner failed to account for the behaviour of
Suballocator, which needs to obtain at least one buffer to allocate any
memory.
Testing:
Added a regression test that caused a crash before the fix.
Updated planner tests.
Was able to run local stress test binary search to completion (it
previously crashed).
Change-Id: I870fbe2f1da01c6123d3716a1198376f9a454c3b
Reviewed-on: http://gerrit.cloudera.org:8080/7871
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/23d79462
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/23d79462
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/23d79462
Branch: refs/heads/release-2.10.0
Commit: 23d79462da5d0108709e8b1399c97606f4ebdf92
Parents: a58394b
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Tue Aug 29 08:29:44 2017 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Wed Aug 30 14:55:08 2017 -0700
----------------------------------------------------------------------
.../apache/impala/planner/AggregationNode.java | 10 ++--
.../queries/PlannerTest/max-row-size.test | 18 +++---
.../queries/PlannerTest/mt-dop-validation.test | 4 +-
.../PlannerTest/resource-requirements.test | 60 ++++++++++----------
.../PlannerTest/spillable-buffer-sizing.test | 38 ++++++-------
.../admission-reject-min-reservation.test | 6 +-
.../queries/QueryTest/aggregation.test | 12 ++++
7 files changed, 81 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/AggregationNode.java b/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
index 2778a7f..c31f448 100644
--- a/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
@@ -323,10 +323,12 @@ public class AggregationNode extends PlanNode {
if (useStreamingPreagg_) {
// We can execute a streaming preagg without any buffers by passing through rows,
// but that is a very low performance mode of execution if the aggregation reduces
- // its input significantly. Instead reserve memory for one buffer and 64kb of hash
- // tables per partition. We don't need to reserve memory for large rows since they
- // can be passed through if needed.
- perInstanceMinReservation = (bufferSize + 64 * 1024) * PARTITION_FANOUT;
+ // its input significantly. Instead reserve memory for one buffer per partition
+ // and at least 64kb for hash tables per partition. We must reserve at least one
+ // full buffer for hash tables for the suballocator to subdivide. We don't need to
+ // reserve memory for large rows since they can be passed through if needed.
+ perInstanceMinReservation = bufferSize * PARTITION_FANOUT +
+ Math.max(64 * 1024 * PARTITION_FANOUT, bufferSize);
} else {
long minBuffers = PARTITION_FANOUT + 1 + (aggInfo_.needsSerialize() ? 1 : 0);
// Two of the buffers need to be buffers large enough to hold the maximum-sized
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test b/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
index 5a0a76c..d563444 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
@@ -150,7 +150,7 @@ from tpch_parquet.lineitem
group by 1, 2
having count(*) = 1
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=110.00MB
+Max Per-Host Resource Reservation: Memory=111.00MB
Per-Host Resource Estimates: Memory=251.12MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -176,11 +176,11 @@ Per-Host Resources: mem-estimate=46.00MB mem-reservation=46.00MB
| tuple-ids=2 row-size=33B cardinality=4690314
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
-Per-Host Resources: mem-estimate=85.12MB mem-reservation=64.00MB
+Per-Host Resources: mem-estimate=85.12MB mem-reservation=65.00MB
03:AGGREGATE [STREAMING]
| output: count(*)
| group by: l_orderkey, o_orderstatus
-| mem-estimate=54.12MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=54.12MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=2 row-size=33B cardinality=4690314
|
02:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -224,7 +224,7 @@ Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
select distinct *
from tpch_parquet.lineitem
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=79.00MB
+Max Per-Host Resource Reservation: Memory=80.00MB
Per-Host Resource Estimates: Memory=3.31GB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -248,10 +248,10 @@ Per-Host Resources: mem-estimate=1.62GB mem-reservation=46.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=1.69GB mem-reservation=33.00MB
+Per-Host Resources: mem-estimate=1.69GB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
-| mem-estimate=1.62GB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
@@ -268,7 +268,7 @@ select l_orderkey, l_partkey, group_concat(l_linestatus, ",")
from tpch_parquet.lineitem
group by 1, 2
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=81.00MB
+Max Per-Host Resource Reservation: Memory=82.00MB
Per-Host Resource Estimates: Memory=482.91MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -293,11 +293,11 @@ Per-Host Resources: mem-estimate=201.46MB mem-reservation=48.00MB
| tuple-ids=1 row-size=32B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=281.46MB mem-reservation=33.00MB
+Per-Host Resources: mem-estimate=281.46MB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| output: group_concat(l_linestatus, ',')
| group by: l_orderkey, l_partkey
-| mem-estimate=201.46MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=201.46MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=32B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test b/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
index 9f80785..dcebf07 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
@@ -95,11 +95,11 @@ Per-Host Resources: mem-estimate=384.00MB mem-reservation=102.00MB
| tuple-ids=1 row-size=16B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9
-Per-Host Resources: mem-estimate=432.00MB mem-reservation=99.00MB
+Per-Host Resources: mem-estimate=432.00MB mem-reservation=102.00MB
01:AGGREGATE [STREAMING]
| output: count(int_col)
| group by: bigint_col
-| mem-estimate=128.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=128.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=16B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
index 90035c1..fe3fd41 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
@@ -375,8 +375,8 @@ PLAN-ROOT SINK
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=41.50MB
-Per-Host Resource Estimates: Memory=123.00MB
+Max Per-Host Resource Reservation: Memory=42.50MB
+Per-Host Resource Estimates: Memory=124.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -400,11 +400,11 @@ Per-Host Resources: mem-estimate=10.00MB mem-reservation=8.50MB
| tuple-ids=1 row-size=16B cardinality=1563438
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=113.00MB mem-reservation=33.00MB
+Per-Host Resources: mem-estimate=114.00MB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: l_orderkey
-| mem-estimate=33.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=16B cardinality=1563438
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
@@ -415,8 +415,8 @@ Per-Host Resources: mem-estimate=113.00MB mem-reservation=33.00MB
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=75.50MB
-Per-Host Resource Estimates: Memory=246.00MB
+Max Per-Host Resource Reservation: Memory=77.50MB
+Per-Host Resource Estimates: Memory=248.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -440,11 +440,11 @@ Per-Host Resources: mem-estimate=20.00MB mem-reservation=9.50MB
| tuple-ids=1 row-size=16B cardinality=1563438
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=226.00MB mem-reservation=66.00MB
+Per-Host Resources: mem-estimate=228.00MB mem-reservation=68.00MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: l_orderkey
-| mem-estimate=33.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=16B cardinality=1563438
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
@@ -1548,7 +1548,7 @@ PLAN-ROOT SINK
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=78B cardinality=600122
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=71.75MB
+Max Per-Host Resource Reservation: Memory=72.75MB
Per-Host Resource Estimates: Memory=344.33MB
F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1643,10 +1643,10 @@ Per-Host Resources: mem-estimate=97.00MB mem-reservation=34.00MB
| tuple-ids=2 row-size=70B cardinality=575772
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
-Per-Host Resources: mem-estimate=47.33MB mem-reservation=37.75MB
+Per-Host Resources: mem-estimate=47.33MB mem-reservation=38.75MB
04:AGGREGATE [STREAMING]
| group by: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
-| mem-estimate=42.58MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=42.58MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=2 row-size=70B cardinality=575772
|
03:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -1688,7 +1688,7 @@ Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=78B cardinality=600122
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=139.75MB
+Max Per-Host Resource Reservation: Memory=141.75MB
Per-Host Resource Estimates: Memory=684.91MB
F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1799,10 +1799,10 @@ Per-Host Resources: mem-estimate=194.00MB mem-reservation=68.00MB
| tuple-ids=2 row-size=70B cardinality=575772
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
-Per-Host Resources: mem-estimate=90.91MB mem-reservation=71.75MB
+Per-Host Resources: mem-estimate=90.91MB mem-reservation=73.75MB
04:AGGREGATE [STREAMING]
| group by: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
-| mem-estimate=42.58MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=42.58MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=2 row-size=70B cardinality=575772
|
03:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -1968,7 +1968,7 @@ PLAN-ROOT SINK
mem-estimate=88.00MB mem-reservation=0B
tuple-ids=2 row-size=16B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=150.12MB
+Max Per-Host Resource Reservation: Memory=152.12MB
Per-Host Resource Estimates: Memory=511.55MB
F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2000,11 +2000,11 @@ Per-Host Resources: mem-estimate=60.41MB mem-reservation=34.00MB
| tuple-ids=6 row-size=100B cardinality=575772
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
-Per-Host Resources: mem-estimate=115.78MB mem-reservation=83.12MB
+Per-Host Resources: mem-estimate=115.78MB mem-reservation=84.12MB
08:AGGREGATE [STREAMING]
| output: sum(l_quantity)
| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
-| mem-estimate=60.40MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=60.40MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=6 row-size=100B cardinality=575772
|
07:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]
@@ -2025,11 +2025,11 @@ Per-Host Resources: mem-estimate=115.78MB mem-reservation=83.12MB
| | tuple-ids=4 row-size=24B cardinality=1563438
| |
| F04:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-| Per-Host Resources: mem-estimate=127.36MB mem-reservation=33.00MB
+| Per-Host Resources: mem-estimate=127.36MB mem-reservation=34.00MB
| 04:AGGREGATE [STREAMING]
| | output: sum(l_quantity)
| | group by: l_orderkey
-| | mem-estimate=39.36MB mem-reservation=33.00MB spill-buffer=2.00MB
+| | mem-estimate=39.36MB mem-reservation=34.00MB spill-buffer=2.00MB
| | tuple-ids=4 row-size=24B cardinality=1563438
| |
| 03:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -2098,7 +2098,7 @@ Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
mem-estimate=88.00MB mem-reservation=0B
tuple-ids=2 row-size=16B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=254.88MB
+Max Per-Host Resource Reservation: Memory=258.88MB
Per-Host Resource Estimates: Memory=967.22MB
F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2130,11 +2130,11 @@ Per-Host Resources: mem-estimate=120.82MB mem-reservation=68.00MB
| tuple-ids=6 row-size=100B cardinality=575772
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
-Per-Host Resources: mem-estimate=175.68MB mem-reservation=120.88MB
+Per-Host Resources: mem-estimate=175.68MB mem-reservation=122.88MB
08:AGGREGATE [STREAMING]
| output: sum(l_quantity)
| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
-| mem-estimate=60.40MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=60.40MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=6 row-size=100B cardinality=575772
|
07:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]
@@ -2163,11 +2163,11 @@ Per-Host Resources: mem-estimate=175.68MB mem-reservation=120.88MB
| | tuple-ids=4 row-size=24B cardinality=1563438
| |
| F04:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-| Per-Host Resources: mem-estimate=254.73MB mem-reservation=66.00MB
+| Per-Host Resources: mem-estimate=254.73MB mem-reservation=68.00MB
| 04:AGGREGATE [STREAMING]
| | output: sum(l_quantity)
| | group by: l_orderkey
-| | mem-estimate=39.36MB mem-reservation=33.00MB spill-buffer=2.00MB
+| | mem-estimate=39.36MB mem-reservation=34.00MB spill-buffer=2.00MB
| | tuple-ids=4 row-size=24B cardinality=1563438
| |
| 03:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -2452,7 +2452,7 @@ PLAN-ROOT SINK
mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=66B cardinality=150000
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=102.94MB
+Max Per-Host Resource Reservation: Memory=103.94MB
Per-Host Resource Estimates: Memory=473.94MB
WARNING: The following tables are missing relevant table and/or column statistics.
tpch_nested_parquet.customer
@@ -2478,10 +2478,10 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=34.00MB
| tuple-ids=6 row-size=58B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=345.94MB mem-reservation=68.94MB
+Per-Host Resources: mem-estimate=345.94MB mem-reservation=69.94MB
09:AGGREGATE [STREAMING]
| group by: c_name, o1.o_orderkey, o2.o_orderstatus
-| mem-estimate=128.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=128.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=6 row-size=58B cardinality=1500000
|
01:SUBPLAN
@@ -2531,7 +2531,7 @@ Per-Host Resources: mem-estimate=345.94MB mem-reservation=68.94MB
mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=66B cardinality=150000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=205.88MB
+Max Per-Host Resource Reservation: Memory=207.88MB
Per-Host Resource Estimates: Memory=947.88MB
WARNING: The following tables are missing relevant table and/or column statistics.
tpch_nested_parquet.customer
@@ -2557,10 +2557,10 @@ Per-Host Resources: mem-estimate=256.00MB mem-reservation=68.00MB
| tuple-ids=6 row-size=58B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=691.88MB mem-reservation=137.88MB
+Per-Host Resources: mem-estimate=691.88MB mem-reservation=139.88MB
09:AGGREGATE [STREAMING]
| group by: c_name, o1.o_orderkey, o2.o_orderstatus
-| mem-estimate=128.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=128.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=6 row-size=58B cardinality=1500000
|
01:SUBPLAN
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
index 115a8bd..719b07d 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
@@ -603,7 +603,7 @@ from tpch_parquet.lineitem
group by 1, 2
having count(*) = 1
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=84.00MB
+Max Per-Host Resource Reservation: Memory=85.00MB
Per-Host Resource Estimates: Memory=225.12MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -629,11 +629,11 @@ Per-Host Resources: mem-estimate=34.00MB mem-reservation=34.00MB
| tuple-ids=2 row-size=33B cardinality=4690314
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
-Per-Host Resources: mem-estimate=71.12MB mem-reservation=50.00MB
+Per-Host Resources: mem-estimate=71.12MB mem-reservation=51.00MB
03:AGGREGATE [STREAMING]
| output: count(*)
| group by: l_orderkey, o_orderstatus
-| mem-estimate=54.12MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=54.12MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=2 row-size=33B cardinality=4690314
|
02:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -672,8 +672,8 @@ Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=117.00MB
-Per-Host Resource Estimates: Memory=357.00MB
+Max Per-Host Resource Reservation: Memory=119.00MB
+Per-Host Resource Estimates: Memory=359.00MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -698,11 +698,11 @@ Per-Host Resources: mem-estimate=34.00MB mem-reservation=34.00MB
| tuple-ids=2 row-size=33B cardinality=4690314
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
-Per-Host Resources: mem-estimate=83.00MB mem-reservation=83.00MB
+Per-Host Resources: mem-estimate=85.00MB mem-reservation=85.00MB
03:AGGREGATE [STREAMING]
| output: count(*)
| group by: l_orderkey, o_orderstatus
-| mem-estimate=33.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=2 row-size=33B cardinality=4690314
|
02:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -753,7 +753,7 @@ Per-Host Resources: mem-estimate=160.00MB mem-reservation=0B
select distinct *
from tpch_parquet.lineitem
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=67.00MB
+Max Per-Host Resource Reservation: Memory=68.00MB
Per-Host Resource Estimates: Memory=3.31GB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -777,10 +777,10 @@ Per-Host Resources: mem-estimate=1.62GB mem-reservation=34.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=1.69GB mem-reservation=33.00MB
+Per-Host Resources: mem-estimate=1.69GB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
-| mem-estimate=1.62GB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
@@ -791,7 +791,7 @@ Per-Host Resources: mem-estimate=1.69GB mem-reservation=33.00MB
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=134.00MB
+Max Per-Host Resource Reservation: Memory=136.00MB
Per-Host Resource Estimates: Memory=6.62GB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -815,10 +815,10 @@ Per-Host Resources: mem-estimate=3.23GB mem-reservation=68.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=3.39GB mem-reservation=66.00MB
+Per-Host Resources: mem-estimate=3.39GB mem-reservation=68.00MB
01:AGGREGATE [STREAMING]
| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
-| mem-estimate=1.62GB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
@@ -834,7 +834,7 @@ select string_col, count(*)
from functional_parquet.alltypestiny
group by string_col
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=67.00MB
+Max Per-Host Resource Reservation: Memory=68.00MB
Per-Host Resource Estimates: Memory=272.00MB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypestiny
@@ -861,11 +861,11 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=34.00MB
| tuple-ids=1 row-size=24B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=144.00MB mem-reservation=33.00MB
+Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: string_col
-| mem-estimate=128.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=128.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=24B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypestiny, RANDOM]
@@ -876,7 +876,7 @@ Per-Host Resources: mem-estimate=144.00MB mem-reservation=33.00MB
mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=16B cardinality=unavailable
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=134.00MB
+Max Per-Host Resource Reservation: Memory=136.00MB
Per-Host Resource Estimates: Memory=544.00MB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypestiny
@@ -903,11 +903,11 @@ Per-Host Resources: mem-estimate=256.00MB mem-reservation=68.00MB
| tuple-ids=1 row-size=24B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=288.00MB mem-reservation=66.00MB
+Per-Host Resources: mem-estimate=288.00MB mem-reservation=68.00MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: string_col
-| mem-estimate=128.00MB mem-reservation=33.00MB spill-buffer=2.00MB
+| mem-estimate=128.00MB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=24B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypestiny, RANDOM]
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
index 940fd6e..a84ec34 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
@@ -5,7 +5,7 @@ select distinct * from functional_parquet.alltypesagg
---- CATCH
minimum memory reservation is greater than memory available to the
query for buffer reservations. Memory reservation needed given the
- current plan: 67.00 MB. Set mem_limit to at least 142.00 MB.
+ current plan: 68.00 MB. Set mem_limit to at least 143.00 MB.
====
---- QUERY
set mem_limit=150mb;
@@ -13,14 +13,14 @@ select distinct * from functional_parquet.alltypesagg
---- CATCH
minimum memory reservation needed is greater than pool max mem resources.
Pool max mem resources: 10.00 MB.
- Cluster-wide memory reservation needed: 201.00 MB
+ Cluster-wide memory reservation needed: 204.00 MB
====
---- QUERY
set buffer_pool_limit=10mb;
select distinct * from functional_parquet.alltypesagg
---- CATCH
minimum memory reservation is greater than memory available to the query
- for buffer reservations. Increase the buffer_pool_limit to 67.00 MB.
+ for buffer reservations. Increase the buffer_pool_limit to 68.00 MB.
====
---- QUERY
set mem_limit=1024;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/23d79462/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/aggregation.test b/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
index de1c507..3cc2f20 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
@@ -1319,3 +1319,15 @@ select 1 from functional_parquet.alltypes having count(*) > 1
---- TYPES
tinyint
====
+---- QUERY
+# IMPALA-5855: pre-aggregation does not reserve enough memory with 2MB buffers.
+# The pre-aggregation in this query is estimated to consume enough memory for the planner
+# to use 2MB buffers.
+set debug_action="-1:PREPARE:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select count(*) from (
+ select distinct l_orderkey, l_comment from tpch_parquet.lineitem) v
+---- RESULTS
+6001198
+---- TYPES
+bigint
+====