You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/04/28 23:41:59 UTC
[08/15] impala git commit: IMPALA-4835: switch I/O buffers to buffer
pool
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test
index e6dfe72..7b70a9d 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test
@@ -2,7 +2,7 @@
select * from functional.alltypes order by random()
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
+| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -19,14 +19,14 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=0B
+ mem-estimate=128.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=97B cardinality=7300
====
# sort on a deterministic expr that exceeds the cost threshold
select * from functional.alltypes order by abs(id) + abs(id)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
+| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -43,14 +43,14 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=0B
+ mem-estimate=128.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=97B cardinality=7300
====
# sort on a deterministic expr that doesn't exceed the cost threshold
select * from functional.alltypes order by tinyint_col + 1
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
+| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -66,7 +66,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=0B
+ mem-estimate=128.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=97B cardinality=7300
====
# sort on multiple exprs, subset is materialized
@@ -74,7 +74,7 @@ select * from functional.alltypes
order by dayofweek(timestamp_col), true, id + 1, string_col = date_string_col, id = tinyint_col
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
+| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -91,7 +91,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=0B
+ mem-estimate=128.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=97B cardinality=7300
====
# expensive analytic order by expr gets materialized
@@ -99,7 +99,7 @@ select last_value(id) over (order by to_date(timestamp_col), bool_col is null)
from functional.alltypes
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.00MB
+| Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.03MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -123,7 +123,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=0B
+ mem-estimate=128.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=21B cardinality=7300
====
# expensive order by expr in top-n gets materialized
@@ -131,7 +131,7 @@ select id from functional.alltypes order by string_col like 'a.*b', id * bigint_
regexp_replace(string_col, 'a.*b', 'c') limit 10
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -148,14 +148,14 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=0B
+ mem-estimate=128.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=29B cardinality=7300
====
# sort on udf, gets materialized
select * from functional.alltypes order by TestFn(double_col)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
+| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -172,14 +172,14 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=0B
+ mem-estimate=128.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=97B cardinality=7300
====
# sort expr contains SlotRefs that don't need to be materialized separately
select concat(date_string_col, string_col) c from functional.alltypes order by c
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
+| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -196,7 +196,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=0B
+ mem-estimate=128.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=41B cardinality=7300
====
# IMPALA-5270: Rand() and udf inside inline view referenced by analytic function.
@@ -237,6 +237,6 @@ PLAN-ROOT SINK
partitions: 4/4 rows=8
columns: all
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=0B
+ mem-estimate=32.00MB mem-reservation=8.00KB
tuple-ids=0 row-size=12B cardinality=8
====
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
index 03273bc..f57e578 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
@@ -3,7 +3,7 @@ select straight_join *
from tpch_parquet.customer
inner join tpch_parquet.nation on c_nationkey = n_nationkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=2.94MB
+Max Per-Host Resource Reservation: Memory=18.97MB
Per-Host Resource Estimates: Memory=42.94MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -16,7 +16,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=355B cardinality=150000
|
F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=26.94MB mem-reservation=2.94MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=26.94MB mem-reservation=18.94MB runtime-filters-memory=1.00MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash predicates: c_nationkey = n_nationkey
| fk/pk conjuncts: c_nationkey = n_nationkey
@@ -29,14 +29,14 @@ Per-Host Resources: mem-estimate=26.94MB mem-reservation=2.94MB runtime-filters-
| | tuple-ids=1 row-size=117B cardinality=25
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
| 01:SCAN HDFS [tpch_parquet.nation, RANDOM]
| partitions=1/1 files=1 size=2.74KB
| stored statistics:
| table: rows=25 size=2.74KB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=16.00MB mem-reservation=0B
+| mem-estimate=16.00MB mem-reservation=32.00KB
| tuple-ids=1 row-size=117B cardinality=25
|
00:SCAN HDFS [tpch_parquet.customer, RANDOM]
@@ -46,10 +46,10 @@ Per-Host Resources: mem-estimate=26.94MB mem-reservation=2.94MB runtime-filters-
table: rows=150000 size=12.31MB
columns: all
extrapolated-rows=disabled
- mem-estimate=24.00MB mem-reservation=0B
+ mem-estimate=24.00MB mem-reservation=16.00MB
tuple-ids=0 row-size=238B cardinality=150000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=5.88MB
+Max Per-Host Resource Reservation: Memory=37.94MB
Per-Host Resource Estimates: Memory=85.88MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -62,7 +62,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=355B cardinality=150000
|
F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-Per-Host Resources: mem-estimate=53.88MB mem-reservation=5.88MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=53.88MB mem-reservation=37.88MB runtime-filters-memory=1.00MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash-table-id=00
| hash predicates: c_nationkey = n_nationkey
@@ -83,14 +83,14 @@ Per-Host Resources: mem-estimate=53.88MB mem-reservation=5.88MB runtime-filters-
| | tuple-ids=1 row-size=117B cardinality=25
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=64.00KB
| 01:SCAN HDFS [tpch_parquet.nation, RANDOM]
| partitions=1/1 files=1 size=2.74KB
| stored statistics:
| table: rows=25 size=2.74KB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=16.00MB mem-reservation=0B
+| mem-estimate=16.00MB mem-reservation=32.00KB
| tuple-ids=1 row-size=117B cardinality=25
|
00:SCAN HDFS [tpch_parquet.customer, RANDOM]
@@ -100,7 +100,7 @@ Per-Host Resources: mem-estimate=53.88MB mem-reservation=5.88MB runtime-filters-
table: rows=150000 size=12.31MB
columns: all
extrapolated-rows=disabled
- mem-estimate=24.00MB mem-reservation=0B
+ mem-estimate=24.00MB mem-reservation=16.00MB
tuple-ids=0 row-size=238B cardinality=150000
====
# Join with large build side - should use default-sized buffers.
@@ -108,8 +108,8 @@ select straight_join *
from tpch_parquet.lineitem
left join tpch_parquet.orders on l_orderkey = o_orderkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=34.00MB
-Per-Host Resource Estimates: Memory=420.41MB
+Max Per-Host Resource Reservation: Memory=154.00MB
+Per-Host Resource Estimates: Memory=428.41MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -121,7 +121,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1N row-size=454B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=380.41MB mem-reservation=34.00MB
+Per-Host Resources: mem-estimate=380.41MB mem-reservation=106.00MB
02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
| hash predicates: l_orderkey = o_orderkey
| fk/pk conjuncts: l_orderkey = o_orderkey
@@ -133,27 +133,27 @@ Per-Host Resources: mem-estimate=380.41MB mem-reservation=34.00MB
| | tuple-ids=1 row-size=191B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-| Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=48.00MB mem-reservation=48.00MB
| 01:SCAN HDFS [tpch_parquet.orders, RANDOM]
| partitions=1/1 files=2 size=54.07MB
| stored statistics:
| table: rows=1500000 size=54.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=40.00MB mem-reservation=0B
+| mem-estimate=48.00MB mem-reservation=48.00MB
| tuple-ids=1 row-size=191B cardinality=1500000
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.73MB
+ partitions=1/1 files=3 size=193.72MB
stored statistics:
- table: rows=6001215 size=193.73MB
+ table: rows=6001215 size=193.72MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=0B
+ mem-estimate=80.00MB mem-reservation=72.00MB
tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=68.00MB
-Per-Host Resource Estimates: Memory=840.83MB
+Max Per-Host Resource Reservation: Memory=308.00MB
+Per-Host Resource Estimates: Memory=856.83MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -165,7 +165,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1N row-size=454B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=760.83MB mem-reservation=68.00MB
+Per-Host Resources: mem-estimate=760.83MB mem-reservation=212.00MB
02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
| hash-table-id=00
| hash predicates: l_orderkey = o_orderkey
@@ -185,23 +185,23 @@ Per-Host Resources: mem-estimate=760.83MB mem-reservation=68.00MB
| | tuple-ids=1 row-size=191B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-| Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=96.00MB mem-reservation=96.00MB
| 01:SCAN HDFS [tpch_parquet.orders, RANDOM]
| partitions=1/1 files=2 size=54.07MB
| stored statistics:
| table: rows=1500000 size=54.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=40.00MB mem-reservation=0B
+| mem-estimate=48.00MB mem-reservation=48.00MB
| tuple-ids=1 row-size=191B cardinality=1500000
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.73MB
+ partitions=1/1 files=3 size=193.72MB
stored statistics:
- table: rows=6001215 size=193.73MB
+ table: rows=6001215 size=193.72MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=0B
+ mem-estimate=80.00MB mem-reservation=72.00MB
tuple-ids=0 row-size=263B cardinality=6001215
====
# Shuffle join with mid-sized input.
@@ -209,8 +209,8 @@ select straight_join *
from tpch_parquet.orders
join /*+shuffle*/ tpch_parquet.customer on o_custkey = c_custkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=36.00MB
-Per-Host Resource Estimates: Memory=100.00MB
+Max Per-Host Resource Reservation: Memory=100.00MB
+Per-Host Resource Estimates: Memory=108.00MB
F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -235,14 +235,14 @@ Per-Host Resources: mem-estimate=35.00MB mem-reservation=35.00MB runtime-filters
| | tuple-ids=1 row-size=238B cardinality=150000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=24.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=24.00MB mem-reservation=16.00MB
| 01:SCAN HDFS [tpch_parquet.customer, RANDOM]
| partitions=1/1 files=1 size=12.31MB
| stored statistics:
| table: rows=150000 size=12.31MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=24.00MB mem-reservation=0B
+| mem-estimate=24.00MB mem-reservation=16.00MB
| tuple-ids=1 row-size=238B cardinality=150000
|
03:EXCHANGE [HASH(o_custkey)]
@@ -250,7 +250,7 @@ Per-Host Resources: mem-estimate=35.00MB mem-reservation=35.00MB runtime-filters
| tuple-ids=0 row-size=191B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-Per-Host Resources: mem-estimate=41.00MB mem-reservation=1.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=49.00MB mem-reservation=49.00MB runtime-filters-memory=1.00MB
00:SCAN HDFS [tpch_parquet.orders, RANDOM]
partitions=1/1 files=2 size=54.07MB
runtime filters: RF000[bloom] -> o_custkey
@@ -258,11 +258,11 @@ Per-Host Resources: mem-estimate=41.00MB mem-reservation=1.00MB runtime-filters-
table: rows=1500000 size=54.07MB
columns: all
extrapolated-rows=disabled
- mem-estimate=40.00MB mem-reservation=0B
+ mem-estimate=48.00MB mem-reservation=48.00MB
tuple-ids=0 row-size=191B cardinality=1500000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=38.00MB
-Per-Host Resource Estimates: Memory=166.00MB
+Max Per-Host Resource Reservation: Memory=166.00MB
+Per-Host Resource Estimates: Memory=182.00MB
F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -295,14 +295,14 @@ Per-Host Resources: mem-estimate=36.00MB mem-reservation=36.00MB runtime-filters
| | tuple-ids=1 row-size=238B cardinality=150000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-| Per-Host Resources: mem-estimate=48.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=48.00MB mem-reservation=32.00MB
| 01:SCAN HDFS [tpch_parquet.customer, RANDOM]
| partitions=1/1 files=1 size=12.31MB
| stored statistics:
| table: rows=150000 size=12.31MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=24.00MB mem-reservation=0B
+| mem-estimate=24.00MB mem-reservation=16.00MB
| tuple-ids=1 row-size=238B cardinality=150000
|
03:EXCHANGE [HASH(o_custkey)]
@@ -310,7 +310,7 @@ Per-Host Resources: mem-estimate=36.00MB mem-reservation=36.00MB runtime-filters
| tuple-ids=0 row-size=191B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-Per-Host Resources: mem-estimate=82.00MB mem-reservation=2.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=98.00MB mem-reservation=98.00MB runtime-filters-memory=1.00MB
00:SCAN HDFS [tpch_parquet.orders, RANDOM]
partitions=1/1 files=2 size=54.07MB
runtime filters: RF000[bloom] -> o_custkey
@@ -318,7 +318,7 @@ Per-Host Resources: mem-estimate=82.00MB mem-reservation=2.00MB runtime-filters-
table: rows=1500000 size=54.07MB
columns: all
extrapolated-rows=disabled
- mem-estimate=40.00MB mem-reservation=0B
+ mem-estimate=48.00MB mem-reservation=48.00MB
tuple-ids=0 row-size=191B cardinality=1500000
====
# Broadcast join with mid-sized input - should use larger buffers than shuffle join.
@@ -326,8 +326,8 @@ select straight_join *
from tpch_parquet.orders
join /*+broadcast*/ tpch_parquet.customer on o_custkey = c_custkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=35.00MB
-Per-Host Resource Estimates: Memory=102.38MB
+Max Per-Host Resource Reservation: Memory=99.00MB
+Per-Host Resource Estimates: Memory=110.38MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -339,7 +339,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=428B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-Per-Host Resources: mem-estimate=78.38MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=86.38MB mem-reservation=83.00MB runtime-filters-memory=1.00MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash predicates: o_custkey = c_custkey
| fk/pk conjuncts: o_custkey = c_custkey
@@ -352,14 +352,14 @@ Per-Host Resources: mem-estimate=78.38MB mem-reservation=35.00MB runtime-filters
| | tuple-ids=1 row-size=238B cardinality=150000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=24.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=24.00MB mem-reservation=16.00MB
| 01:SCAN HDFS [tpch_parquet.customer, RANDOM]
| partitions=1/1 files=1 size=12.31MB
| stored statistics:
| table: rows=150000 size=12.31MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=24.00MB mem-reservation=0B
+| mem-estimate=24.00MB mem-reservation=16.00MB
| tuple-ids=1 row-size=238B cardinality=150000
|
00:SCAN HDFS [tpch_parquet.orders, RANDOM]
@@ -369,11 +369,11 @@ Per-Host Resources: mem-estimate=78.38MB mem-reservation=35.00MB runtime-filters
table: rows=1500000 size=54.07MB
columns: all
extrapolated-rows=disabled
- mem-estimate=40.00MB mem-reservation=0B
+ mem-estimate=48.00MB mem-reservation=48.00MB
tuple-ids=0 row-size=191B cardinality=1500000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=70.00MB
-Per-Host Resource Estimates: Memory=204.76MB
+Max Per-Host Resource Reservation: Memory=198.00MB
+Per-Host Resource Estimates: Memory=220.76MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -385,7 +385,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=428B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-Per-Host Resources: mem-estimate=156.76MB mem-reservation=70.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=172.76MB mem-reservation=166.00MB runtime-filters-memory=1.00MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash-table-id=00
| hash predicates: o_custkey = c_custkey
@@ -406,14 +406,14 @@ Per-Host Resources: mem-estimate=156.76MB mem-reservation=70.00MB runtime-filter
| | tuple-ids=1 row-size=238B cardinality=150000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-| Per-Host Resources: mem-estimate=48.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=48.00MB mem-reservation=32.00MB
| 01:SCAN HDFS [tpch_parquet.customer, RANDOM]
| partitions=1/1 files=1 size=12.31MB
| stored statistics:
| table: rows=150000 size=12.31MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=24.00MB mem-reservation=0B
+| mem-estimate=24.00MB mem-reservation=16.00MB
| tuple-ids=1 row-size=238B cardinality=150000
|
00:SCAN HDFS [tpch_parquet.orders, RANDOM]
@@ -423,7 +423,7 @@ Per-Host Resources: mem-estimate=156.76MB mem-reservation=70.00MB runtime-filter
table: rows=1500000 size=54.07MB
columns: all
extrapolated-rows=disabled
- mem-estimate=40.00MB mem-reservation=0B
+ mem-estimate=48.00MB mem-reservation=48.00MB
tuple-ids=0 row-size=191B cardinality=1500000
====
# Join with no stats for right input - should use default buffers.
@@ -431,7 +431,7 @@ select straight_join *
from functional_parquet.alltypes
left join functional_parquet.alltypestiny on alltypes.id = alltypestiny.id
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=34.00MB
+Max Per-Host Resource Reservation: Memory=34.17MB
Per-Host Resource Estimates: Memory=2.03GB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypes, functional_parquet.alltypestiny
@@ -446,7 +446,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1N row-size=176B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.00MB
+Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.09MB
02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
| hash predicates: alltypes.id = alltypestiny.id
| fk/pk conjuncts: assumed fk/pk
@@ -458,7 +458,7 @@ Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.00MB
| | tuple-ids=1 row-size=88B cardinality=unavailable
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=16.00MB mem-reservation=88.00KB
| 01:SCAN HDFS [functional_parquet.alltypestiny, RANDOM]
| partitions=4/4 files=4 size=9.75KB
| stored statistics:
@@ -466,7 +466,7 @@ Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.00MB
| partitions: 0/4 rows=unavailable
| columns: unavailable
| extrapolated-rows=disabled
-| mem-estimate=16.00MB mem-reservation=0B
+| mem-estimate=16.00MB mem-reservation=88.00KB
| tuple-ids=1 row-size=88B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
@@ -476,10 +476,10 @@ Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.00MB
partitions: 0/24 rows=unavailable
columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=0B
+ mem-estimate=16.00MB mem-reservation=88.00KB
tuple-ids=0 row-size=88B cardinality=unavailable
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=68.00MB
+Max Per-Host Resource Reservation: Memory=68.34MB
Per-Host Resource Estimates: Memory=4.06GB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypestiny
@@ -494,7 +494,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1N row-size=176B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.00MB
+Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.17MB
02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
| hash-table-id=00
| hash predicates: alltypes.id = alltypestiny.id
@@ -514,7 +514,7 @@ Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.00MB
| | tuple-ids=1 row-size=88B cardinality=unavailable
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=176.00KB
| 01:SCAN HDFS [functional_parquet.alltypestiny, RANDOM]
| partitions=4/4 files=4 size=9.75KB
| stored statistics:
@@ -522,7 +522,7 @@ Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.00MB
| partitions: 0/4 rows=unavailable
| columns: unavailable
| extrapolated-rows=disabled
-| mem-estimate=16.00MB mem-reservation=0B
+| mem-estimate=16.00MB mem-reservation=88.00KB
| tuple-ids=1 row-size=88B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
@@ -532,7 +532,7 @@ Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.00MB
partitions: 0/24 rows=unavailable
columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=0B
+ mem-estimate=16.00MB mem-reservation=88.00KB
tuple-ids=0 row-size=88B cardinality=unavailable
====
# Low NDV aggregation - should scale down buffers to minimum.
@@ -540,7 +540,7 @@ select c_nationkey, avg(c_acctbal)
from tpch_parquet.customer
group by c_nationkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=3.94MB
+Max Per-Host Resource Reservation: Memory=19.94MB
Per-Host Resource Estimates: Memory=44.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -565,7 +565,7 @@ Per-Host Resources: mem-estimate=10.00MB mem-reservation=1.94MB
| tuple-ids=1 row-size=10B cardinality=25
|
F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=34.00MB mem-reservation=2.00MB
+Per-Host Resources: mem-estimate=34.00MB mem-reservation=18.00MB
01:AGGREGATE [STREAMING]
| output: avg(c_acctbal)
| group by: c_nationkey
@@ -578,10 +578,10 @@ Per-Host Resources: mem-estimate=34.00MB mem-reservation=2.00MB
table: rows=150000 size=12.31MB
columns: all
extrapolated-rows=disabled
- mem-estimate=24.00MB mem-reservation=0B
+ mem-estimate=24.00MB mem-reservation=16.00MB
tuple-ids=0 row-size=10B cardinality=150000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=7.88MB
+Max Per-Host Resource Reservation: Memory=39.88MB
Per-Host Resource Estimates: Memory=88.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -606,7 +606,7 @@ Per-Host Resources: mem-estimate=20.00MB mem-reservation=3.88MB
| tuple-ids=1 row-size=10B cardinality=25
|
F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-Per-Host Resources: mem-estimate=68.00MB mem-reservation=4.00MB
+Per-Host Resources: mem-estimate=68.00MB mem-reservation=36.00MB
01:AGGREGATE [STREAMING]
| output: avg(c_acctbal)
| group by: c_nationkey
@@ -619,7 +619,7 @@ Per-Host Resources: mem-estimate=68.00MB mem-reservation=4.00MB
table: rows=150000 size=12.31MB
columns: all
extrapolated-rows=disabled
- mem-estimate=24.00MB mem-reservation=0B
+ mem-estimate=24.00MB mem-reservation=16.00MB
tuple-ids=0 row-size=10B cardinality=150000
====
# Mid NDV aggregation - should scale down buffers to intermediate size.
@@ -629,7 +629,7 @@ from tpch_parquet.lineitem
group by 1, 2
having count(*) = 1
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=87.00MB
+Max Per-Host Resource Reservation: Memory=111.00MB
Per-Host Resource Estimates: Memory=227.12MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -674,14 +674,14 @@ Per-Host Resources: mem-estimate=72.12MB mem-reservation=52.00MB runtime-filters
| | tuple-ids=1 row-size=25B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-| Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=40.00MB mem-reservation=16.00MB
| 01:SCAN HDFS [tpch_parquet.orders, RANDOM]
| partitions=1/1 files=2 size=54.07MB
| stored statistics:
| table: rows=1500000 size=54.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=40.00MB mem-reservation=0B
+| mem-estimate=40.00MB mem-reservation=16.00MB
| tuple-ids=1 row-size=25B cardinality=1500000
|
04:EXCHANGE [HASH(l_orderkey)]
@@ -689,18 +689,18 @@ Per-Host Resources: mem-estimate=72.12MB mem-reservation=52.00MB runtime-filters
| tuple-ids=0 row-size=8B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=81.00MB mem-reservation=1.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=81.00MB mem-reservation=9.00MB runtime-filters-memory=1.00MB
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.73MB
+ partitions=1/1 files=3 size=193.72MB
runtime filters: RF000[bloom] -> l_orderkey
stored statistics:
- table: rows=6001215 size=193.73MB
+ table: rows=6001215 size=193.72MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=0B
+ mem-estimate=80.00MB mem-reservation=8.00MB
tuple-ids=0 row-size=8B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=123.00MB
+Max Per-Host Resource Reservation: Memory=171.00MB
Per-Host Resource Estimates: Memory=363.00MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -753,14 +753,14 @@ Per-Host Resources: mem-estimate=87.00MB mem-reservation=87.00MB runtime-filters
| | tuple-ids=1 row-size=25B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-| Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=80.00MB mem-reservation=32.00MB
| 01:SCAN HDFS [tpch_parquet.orders, RANDOM]
| partitions=1/1 files=2 size=54.07MB
| stored statistics:
| table: rows=1500000 size=54.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=40.00MB mem-reservation=0B
+| mem-estimate=40.00MB mem-reservation=16.00MB
| tuple-ids=1 row-size=25B cardinality=1500000
|
04:EXCHANGE [HASH(l_orderkey)]
@@ -768,22 +768,22 @@ Per-Host Resources: mem-estimate=87.00MB mem-reservation=87.00MB runtime-filters
| tuple-ids=0 row-size=8B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=162.00MB mem-reservation=2.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=162.00MB mem-reservation=18.00MB runtime-filters-memory=1.00MB
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.73MB
+ partitions=1/1 files=3 size=193.72MB
runtime filters: RF000[bloom] -> l_orderkey
stored statistics:
- table: rows=6001215 size=193.73MB
+ table: rows=6001215 size=193.72MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=0B
+ mem-estimate=80.00MB mem-reservation=8.00MB
tuple-ids=0 row-size=8B cardinality=6001215
====
# High NDV aggregation - should use default buffer size.
select distinct *
from tpch_parquet.lineitem
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=68.00MB
+Max Per-Host Resource Reservation: Memory=140.00MB
Per-Host Resource Estimates: Memory=3.31GB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -807,22 +807,22 @@ Per-Host Resources: mem-estimate=1.62GB mem-reservation=34.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=1.69GB mem-reservation=34.00MB
+Per-Host Resources: mem-estimate=1.69GB mem-reservation=106.00MB
01:AGGREGATE [STREAMING]
| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.73MB
+ partitions=1/1 files=3 size=193.72MB
stored statistics:
- table: rows=6001215 size=193.73MB
+ table: rows=6001215 size=193.72MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=0B
+ mem-estimate=80.00MB mem-reservation=72.00MB
tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=136.00MB
+Max Per-Host Resource Reservation: Memory=280.00MB
Per-Host Resource Estimates: Memory=6.62GB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -846,19 +846,19 @@ Per-Host Resources: mem-estimate=3.23GB mem-reservation=68.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=3.39GB mem-reservation=68.00MB
+Per-Host Resources: mem-estimate=3.39GB mem-reservation=212.00MB
01:AGGREGATE [STREAMING]
| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.73MB
+ partitions=1/1 files=3 size=193.72MB
stored statistics:
- table: rows=6001215 size=193.73MB
+ table: rows=6001215 size=193.72MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=0B
+ mem-estimate=80.00MB mem-reservation=72.00MB
tuple-ids=0 row-size=263B cardinality=6001215
====
# Aggregation with unknown input - should use default buffer size.
@@ -866,7 +866,7 @@ select string_col, count(*)
from functional_parquet.alltypestiny
group by string_col
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=68.00MB
+Max Per-Host Resource Reservation: Memory=68.01MB
Per-Host Resource Estimates: Memory=272.00MB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypestiny
@@ -893,7 +893,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=34.00MB
| tuple-ids=1 row-size=24B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.00MB
+Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.01MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: string_col
@@ -907,10 +907,10 @@ Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.00MB
partitions: 0/4 rows=unavailable
columns: unavailable
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=0B
+ mem-estimate=16.00MB mem-reservation=8.00KB
tuple-ids=0 row-size=16B cardinality=unavailable
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=136.00MB
+Max Per-Host Resource Reservation: Memory=136.02MB
Per-Host Resource Estimates: Memory=544.00MB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypestiny
@@ -937,7 +937,7 @@ Per-Host Resources: mem-estimate=256.00MB mem-reservation=68.00MB
| tuple-ids=1 row-size=24B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=288.00MB mem-reservation=68.00MB
+Per-Host Resources: mem-estimate=288.00MB mem-reservation=68.02MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: string_col
@@ -951,6 +951,6 @@ Per-Host Resources: mem-estimate=288.00MB mem-reservation=68.00MB
partitions: 0/4 rows=unavailable
columns: unavailable
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=0B
+ mem-estimate=16.00MB mem-reservation=8.00KB
tuple-ids=0 row-size=16B cardinality=unavailable
====
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
index 48163ec..8e0b058 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
@@ -2,7 +2,7 @@
select * from functional.alltypes tablesample system(10) repeatable(1234)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=32.00KB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -13,14 +13,14 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=0B
+ mem-estimate=32.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=97B cardinality=730
====
# Sample 50%
select * from functional.alltypes tablesample system(50) repeatable(1234)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=80.00MB mem-reservation=32.00KB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -31,7 +31,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=0B
+ mem-estimate=80.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=97B cardinality=3650
====
# Sampling and scan predicates. Scan predicates are evaluated after sampling and
@@ -40,7 +40,7 @@ select * from functional.alltypes tablesample system(50) repeatable(1234)
where id < 10
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=80.00MB mem-reservation=32.00KB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -53,7 +53,7 @@ PLAN-ROOT SINK
columns: all
extrapolated-rows=disabled
parquet dictionary predicates: id < 10
- mem-estimate=80.00MB mem-reservation=0B
+ mem-estimate=80.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=97B cardinality=365
====
# Partition pruning + sampling. Partition pruning happens before sampling.
@@ -61,7 +61,7 @@ select * from functional.alltypes tablesample system(50) repeatable(1234)
where year = 2009
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=48.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=48.00MB mem-reservation=32.00KB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -72,7 +72,7 @@ PLAN-ROOT SINK
partitions: 12/12 rows=3650
columns: all
extrapolated-rows=disabled
- mem-estimate=48.00MB mem-reservation=0B
+ mem-estimate=48.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=97B cardinality=1825
====
# Edge case: sample 0%, no files should be selected
@@ -97,7 +97,7 @@ PLAN-ROOT SINK
select * from functional.alltypes tablesample system(1) repeatable(1234)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=32.00KB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -108,7 +108,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=0B
+ mem-estimate=32.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=97B cardinality=73
====
# Edge case: sample 1% and prune partitions, at least one file should be selected
@@ -116,7 +116,7 @@ select * from functional.alltypes tablesample system(1) repeatable(1234)
where year = 2010
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=32.00KB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -127,14 +127,14 @@ PLAN-ROOT SINK
partitions: 12/12 rows=3650
columns: all
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=0B
+ mem-estimate=32.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=97B cardinality=37
====
# Edge case: sample 100%, all files should be selected
select * from functional.alltypes tablesample system (100)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -145,25 +145,25 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=0B
+ mem-estimate=128.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=97B cardinality=7300
====
# Table that has no stats.
select id from functional_parquet.alltypes tablesample system(10) repeatable(1234)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=16.00MB mem-reservation=16.00KB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
00:SCAN HDFS [functional_parquet.alltypes]
- partitions=3/24 files=3 size=22.07KB
+ partitions=3/24 files=3 size=23.96KB
stored statistics:
table: rows=unavailable size=unavailable
partitions: 0/24 rows=unavailable
columns: unavailable
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=0B
+ mem-estimate=16.00MB mem-reservation=16.00KB
tuple-ids=0 row-size=4B cardinality=unavailable
====
# Sampling in a subquery.
@@ -172,7 +172,7 @@ select id from functional.alltypes t1 where exists (
where t1.id = t2.id)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=162.94MB mem-reservation=2.94MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=162.94MB mem-reservation=2.98MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -189,7 +189,7 @@ PLAN-ROOT SINK
| partitions: 4/4 rows=100
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=32.00MB mem-reservation=0B
+| mem-estimate=32.00MB mem-reservation=8.00KB
| tuple-ids=1 row-size=4B cardinality=10
|
00:SCAN HDFS [functional.alltypes t1]
@@ -200,7 +200,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=0B
+ mem-estimate=128.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=4B cardinality=7300
====
# Sampling in WITH-clause view.
@@ -208,7 +208,7 @@ with t as (select * from functional.alltypes tablesample system(10) repeatable(1
select id from t
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=32.00KB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -219,6 +219,6 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=0B
+ mem-estimate=32.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=4B cardinality=730
====
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-planner/queries/PlannerTest/union.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/union.test b/testdata/workloads/functional-planner/queries/PlannerTest/union.test
index a0d0c26..09ac11b 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/union.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/union.test
@@ -3233,7 +3233,7 @@ PLAN-ROOT SINK
| tuple-ids=2 row-size=4B cardinality=7300
|
F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
+Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
00:UNION
| pass-through-operands: all
| mem-estimate=0B mem-reservation=0B
@@ -3246,7 +3246,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
| partitions: 24/24 rows=7300
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=128.00MB mem-reservation=0B
+| mem-estimate=128.00MB mem-reservation=32.00KB
| tuple-ids=1 row-size=4B cardinality=7300
|
01:SCAN HDFS [functional.emptytable, RANDOM]
@@ -3277,7 +3277,7 @@ PLAN-ROOT SINK
| tuple-ids=2 row-size=4B cardinality=7300
|
F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
+Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
00:UNION
| pass-through-operands: all
| mem-estimate=0B mem-reservation=0B
@@ -3300,6 +3300,6 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=0B
+ mem-estimate=128.00MB mem-reservation=32.00KB
tuple-ids=0 row-size=4B cardinality=7300
====
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
index 119c2fd..de12db1 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
@@ -5,7 +5,7 @@ select distinct * from functional_parquet.alltypesagg
---- CATCH
minimum memory reservation is greater than memory available to the
query for buffer reservations. Memory reservation needed given the
- current plan: 68.00 MB. Set mem_limit to at least 143.00 MB.
+ current plan: 68.09 MB. Set mem_limit to at least 100.09 MB.
====
---- QUERY
set mem_limit=150mb;
@@ -13,14 +13,14 @@ select distinct * from functional_parquet.alltypesagg
---- CATCH
minimum memory reservation needed is greater than pool max mem resources.
Pool max mem resources: 10.00 MB.
- Cluster-wide memory reservation needed: 204.00 MB
+ Cluster-wide memory reservation needed: 204.26 MB
====
---- QUERY
set buffer_pool_limit=10mb;
select distinct * from functional_parquet.alltypesagg
---- CATCH
minimum memory reservation is greater than memory available to the query
- for buffer reservations. Increase the buffer_pool_limit to 68.00 MB.
+ for buffer reservations. Increase the buffer_pool_limit to 68.09 MB.
====
---- QUERY
set mem_limit=1024;
@@ -29,14 +29,14 @@ from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
---- CATCH
minimum memory reservation is greater than memory available to the
query for buffer reservations. Memory reservation needed given the
- current plan: 6.75 MB. Set mem_limit to at least 81.75 MB.
+ current plan: 22.75 MB. Set mem_limit to at least 54.75 MB.
====
---- QUERY
-set mem_limit=80mb;
+set mem_limit=50mb;
select count(*)
from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
---- CATCH
minimum memory reservation is greater than memory available to the
query for buffer reservations. Memory reservation needed given the
- current plan: 18.00 MB. Set mem_limit to at least 93.00 MB.
+ current plan: 34.00 MB. Set mem_limit to at least 66.00 MB.
====
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test b/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
index fe0e358..adf1320 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
@@ -1584,9 +1584,10 @@ int, bigint, bigint, double
====
---- QUERY
# Check that the a large analytic query can succeed with the minimum buffers (3 buffers
-# for sort, 2 buffer for analytic).
+# for sort, 2 buffer for analytic, plus 32KB extra for the I/O buffers required to scan
+# the < 16KB functional_parquet.alltypes files).
SET default_spillable_buffer_size=8m;
-SET buffer_pool_limit=40m;
+SET buffer_pool_limit=47m;
SELECT lag(-180, 13) over (ORDER BY t1.int_col ASC, t2.int_col ASC) AS int_col
FROM functional_parquet.alltypes t1 CROSS JOIN functional_parquet.alltypes t2 LIMIT 10;
---- TYPES
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test b/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test
index dea6066..f7013ae 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test
@@ -1,9 +1,8 @@
=====
---- QUERY
+set exec_single_node_rows_threshold=0;
set mem_limit=100k;
-select *
-from alltypes
-where substr(string_col, 1) = "";
+with t as (values(1), (2), (3), (4)) select min(t1.`1` + t2.`1`) from t t1 join t t2;
---- CATCH
Codegen failed to reserve
=====
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test b/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test
index 2c4fede..48649e5 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test
@@ -1,6 +1,6 @@
====
---- QUERY
-set buffer_pool_limit=10m;
+set buffer_pool_limit=90m;
set default_spillable_buffer_size=64k;
select count(*)
from (select distinct o_orderdate, o_custkey, o_comment
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
index 92abd5f..53afc32 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
@@ -5,8 +5,8 @@ explain
select *
from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
---- RESULTS: VERIFY_IS_EQUAL
-'Max Per-Host Resource Reservation: Memory=35.00MB'
-'Per-Host Resource Estimates: Memory=477.41MB'
+row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.*
+row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
''
'PLAN-ROOT SINK'
'04:EXCHANGE [UNPARTITIONED]'
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
index e65745a..6d58d29 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
@@ -5,8 +5,8 @@ explain
select *
from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
---- RESULTS: VERIFY_IS_EQUAL
-'Max Per-Host Resource Reservation: Memory=35.00MB'
-'Per-Host Resource Estimates: Memory=477.41MB'
+row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.*
+row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
''
'PLAN-ROOT SINK'
'|'
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
index b1c58f4..17fdd10 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
@@ -5,8 +5,8 @@ explain
select *
from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
---- RESULTS: VERIFY_IS_EQUAL
-'Max Per-Host Resource Reservation: Memory=35.00MB'
-'Per-Host Resource Estimates: Memory=477.41MB'
+row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.*
+row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
''
'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
'| Per-Host Resources: mem-estimate=0B mem-reservation=0B'
@@ -18,12 +18,12 @@ from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
'| tuple-ids=0,1 row-size=454B cardinality=5757710'
'|'
'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3'
-'Per-Host Resources: mem-estimate=389.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB'
+row_regex:.*Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
'02:HASH JOIN [INNER JOIN, BROADCAST]'
'| hash predicates: l_orderkey = o_orderkey'
'| fk/pk conjuncts: l_orderkey = o_orderkey'
'| runtime filters: RF000[bloom] <- o_orderkey'
-'| mem-estimate=300.41MB mem-reservation=34.00MB spill-buffer=2.00MB'
+row_regex:.*| mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB spill-buffer=[0-9.]*MB.*
'| tuple-ids=0,1 row-size=454B cardinality=5757710'
'|'
'|--03:EXCHANGE [BROADCAST]'
@@ -31,14 +31,14 @@ from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
'| | tuple-ids=1 row-size=191B cardinality=1500000'
'| |'
'| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2'
-'| Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B'
+row_regex:.*| Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
'| 01:SCAN HDFS [tpch.orders, RANDOM]'
row_regex:.*partitions=1/1 files=1 size=.*
'| stored statistics:'
row_regex:.*table: rows=1500000 size=.*
'| columns: all'
'| extrapolated-rows=disabled'
-'| mem-estimate=88.00MB mem-reservation=0B'
+row_regex:.*| mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
'| tuple-ids=1 row-size=191B cardinality=1500000'
'|'
'00:SCAN HDFS [tpch.lineitem, RANDOM]'
@@ -48,7 +48,7 @@ row_regex:.*partitions=1/1 files=1 size=.*
row_regex:.*table: rows=6001215 size=.*
' columns: all'
' extrapolated-rows=disabled'
-' mem-estimate=88.00MB mem-reservation=0B'
+row_regex:.* mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
' tuple-ids=0 row-size=263B cardinality=6001215'
====
---- QUERY
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
index 625e2b0..4ef3524 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
@@ -5,8 +5,8 @@ explain
select *
from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
---- RESULTS: VERIFY_IS_EQUAL
-'Max Per-Host Resource Reservation: Memory=35.00MB'
-'Per-Host Resource Estimates: Memory=477.41MB'
+row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.*
+row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
''
'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
'Per-Host Resources: mem-estimate=0B mem-reservation=0B'
@@ -18,14 +18,14 @@ from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
' tuple-ids=0,1 row-size=454B cardinality=5757710'
''
'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3'
-'Per-Host Resources: mem-estimate=389.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB'
+row_regex:.*Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=04, UNPARTITIONED]'
' | mem-estimate=0B mem-reservation=0B'
' 02:HASH JOIN [INNER JOIN, BROADCAST]'
' | hash predicates: l_orderkey = o_orderkey'
' | fk/pk conjuncts: l_orderkey = o_orderkey'
' | runtime filters: RF000[bloom] <- o_orderkey'
-' | mem-estimate=300.41MB mem-reservation=34.00MB spill-buffer=2.00MB'
+row_regex:.* | mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB spill-buffer=[0-9.]*MB.*
' | tuple-ids=0,1 row-size=454B cardinality=5757710'
' |'
' |--03:EXCHANGE [BROADCAST]'
@@ -39,11 +39,11 @@ row_regex:.*partitions=1/1 files=1 size=.*
row_regex:.*table: rows=6001215 size=.*
' columns: all'
' extrapolated-rows=disabled'
-' mem-estimate=88.00MB mem-reservation=0B'
+row_regex:.* mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
' tuple-ids=0 row-size=263B cardinality=6001215'
''
'F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2'
-'Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B'
+row_regex:.*Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
' DATASTREAM SINK [FRAGMENT=F00, EXCHANGE=03, BROADCAST]'
' | mem-estimate=0B mem-reservation=0B'
' 01:SCAN HDFS [tpch.orders, RANDOM]'
@@ -52,7 +52,7 @@ row_regex:.*partitions=1/1 files=1 size=.*
row_regex:.*table: rows=1500000 size=.*
' columns: all'
' extrapolated-rows=disabled'
-' mem-estimate=88.00MB mem-reservation=0B'
+row_regex:.* mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
' tuple-ids=1 row-size=191B cardinality=1500000'
====
---- QUERY
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
index 8ec3753..44252fb 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
@@ -162,7 +162,7 @@ BIGINT,BIGINT
---- QUERY
# IMPALA-2376
# Set memory limit low enough to get the below query to consistently fail.
-# TODO: change query and/or table so we can consistently check for the following error:
+# This was originally a regression test that hit an error like:
# Failed to allocate buffer for collection '...'.
set mem_limit=4m;
select max(cnt) from customer c,
@@ -170,7 +170,7 @@ select max(cnt) from customer c,
---- TYPES
BIGINT
---- CATCH
-Memory limit exceeded
+Rejected query from pool default-pool: minimum memory reservation is greater than memory available to the query for buffer reservations.
====
---- QUERY
# IMPALA-2473: Scan query with large row size leading to oversized batches.
@@ -237,7 +237,7 @@ bigint, bigint
# This was originally a regression test for IMPALA-5446: dropped status from
# Sorter::Reset() when sort cannot get reserved buffer. However with the
# IMPALA-3200 changes it now succeeds.
-set buffer_pool_limit=100m;
+set buffer_pool_limit=180m;
select c_custkey, c_name, c_address, c_phone, c_acctbal, c_mktsegment, c_comment,
o_orderdate, sum(o_totalprice), min(rnum)
from customer c,
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
index 3d92acb..fdca977 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
@@ -345,7 +345,7 @@ where l_orderkey = 965 and l_extendedprice * l_tax = p_retailprice;
# consumption / spilling behaviour.
####################################################
-SET BUFFER_POOL_LIMIT=40m;
+SET BUFFER_POOL_LIMIT=80m;
SET RUNTIME_FILTER_MODE=GLOBAL;
SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS;
SET RUNTIME_BLOOM_FILTER_SIZE=16M;
@@ -396,15 +396,14 @@ SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS;
SET RUNTIME_FILTER_MIN_SIZE=128MB;
SET RUNTIME_FILTER_MAX_SIZE=500MB;
# Query would have been admitted if memory for runtime filters was not accounted for.
-SET BUFFER_POOL_LIMIT=140MB;
+SET BUFFER_POOL_LIMIT=290MB;
select STRAIGHT_JOIN * from alltypes a join [SHUFFLE] alltypes b
on a.month = b.id and b.int_col = -3
---- RESULTS
---- CATCH
-ImpalaBeeswaxException: INNER EXCEPTION: <class 'beeswaxd.ttypes.BeeswaxException'>
- MESSAGE: Rejected query from pool default-pool: minimum memory reservation is
+Rejected query from pool default-pool: minimum memory reservation is
greater than memory available to the query for buffer reservations. Increase
- the buffer_pool_limit to 290.00 MB. See the query profile for more information
+ the buffer_pool_limit to 290.17 MB. See the query profile for more information
about the per-node memory requirements.
====
---- QUERY
@@ -414,7 +413,7 @@ SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS;
SET RUNTIME_FILTER_MIN_SIZE=128MB;
SET RUNTIME_FILTER_MAX_SIZE=500MB;
# This would run perfectly with just enough memory provided by the buffer pool.
-SET BUFFER_POOL_LIMIT=290MB;
+SET BUFFER_POOL_LIMIT=295MB;
select STRAIGHT_JOIN * from alltypes a join [SHUFFLE] alltypes b
on a.month = b.id and b.int_col = -3
---- RESULTS
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/scanners.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/scanners.test b/testdata/workloads/functional-query/queries/QueryTest/scanners.test
index bd5f496..a000a15 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/scanners.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/scanners.test
@@ -110,3 +110,10 @@ on (v.x = t3.bool_col) where t3.bool_col = true
---- TYPES
BIGINT
====
+---- QUERY
+# Scan an empty unpartitioned table.
+select * from emptytable;
+---- RESULTS
+---- TYPES
+STRING,INT
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/set.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/set.test b/testdata/workloads/functional-query/queries/QueryTest/set.test
index 2779d55..5a2c56a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/set.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/set.test
@@ -108,7 +108,7 @@ select 1
set mem_limit=1;
select count(string_col) from functional.alltypestiny
---- CATCH
-Memory limit exceeded
+Rejected query from pool default-pool: minimum memory reservation
====
---- QUERY
# Set mem_limit back to unlimited and query should succeed again.
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
index 5a58f62..618b828 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
@@ -1,6 +1,6 @@
====
---- QUERY
-set buffer_pool_limit=10m;
+set buffer_pool_limit=34m;
select l_orderkey, count(*)
from lineitem
group by 1
@@ -25,7 +25,7 @@ row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
====
---- QUERY
# Test query with string grouping column and string agg columns
-set buffer_pool_limit=10m;
+set buffer_pool_limit=82m;
set num_nodes=1;
select l_returnflag, l_orderkey, avg(l_tax), min(l_shipmode)
from lineitem
@@ -39,10 +39,9 @@ order by 1,2 limit 3
STRING, BIGINT, DECIMAL, STRING
---- RUNTIME_PROFILE
row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
-row_regex: .*NumRepartitions: .* \([1-9][0-9]*\)
====
---- QUERY
-set buffer_pool_limit=10m;
+set buffer_pool_limit=34m;
select l_orderkey, count(*)
from lineitem
group by 1
@@ -66,7 +65,7 @@ row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
====
---- QUERY
# Test query with string grouping column
-set buffer_pool_limit=10m;
+set buffer_pool_limit=34m;
set num_nodes=1;
select l_comment, count(*)
from lineitem
@@ -82,11 +81,10 @@ order by count(*) desc limit 5
STRING, BIGINT
---- RUNTIME_PROFILE
row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
-row_regex: .*NumRepartitions: .* \([1-9][0-9]*\)
====
---- QUERY
# Test query with string grouping column and string agg columns
-set buffer_pool_limit=10m;
+set buffer_pool_limit=82m;
set num_nodes=1;
select l_returnflag, l_orderkey, round(avg(l_tax),2), min(l_shipmode)
from lineitem
@@ -101,11 +99,10 @@ STRING, BIGINT, DECIMAL, STRING
---- RUNTIME_PROFILE
# Verify that spilling happened in the agg.
row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
-row_regex: .*NumRepartitions: .* \([1-9][0-9]*\)
====
---- QUERY
# Test with non-scalar intermediate state (avg() uses fixed intermediate value).
-set buffer_pool_limit=10m;
+set buffer_pool_limit=34m;
select l_orderkey, avg(l_orderkey)
from lineitem
group by 1
@@ -125,7 +122,7 @@ row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
====
---- QUERY
# Test aggregation spill with group_concat distinct
-set buffer_pool_limit=30m;
+set buffer_pool_limit=78m;
select l_orderkey, count(*), group_concat(distinct l_linestatus, '|')
from lineitem
group by 1
@@ -171,7 +168,7 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
# Merge aggregation requires 17 buffers and preaggregation requires 16 buffers
# plus 1mb of hash tables. The buffer size is 256k for this test.
set max_row_size=256k;
-set buffer_pool_limit=9472k;
+set buffer_pool_limit="57.25M";
select count(*)
from (select distinct * from orders) t
---- TYPES
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test
index 420e721..7678675 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test
@@ -13,7 +13,7 @@
# except:
# * even-numbered l_orderkey values, because there is a NULL on the build side.
# * l_orderkey values > 5500000, because there is a NULL on the probe side
-set buffer_pool_limit=12m;
+set buffer_pool_limit=110m;
select l_orderkey, l_partkey, l_suppkey, l_linenumber
from lineitem
where l_suppkey = 4162 and l_shipmode = 'AIR' and l_returnflag = 'A' and
@@ -54,8 +54,9 @@ order by 1,2,3,4
BIGINT,BIGINT,BIGINT,INT
====
---- QUERY
-set buffer_pool_limit=7m;
-# Execute NAAJ query 1 without enough memory to fit the null build rows in memory.
+set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
+# Execute NAAJ query 1 with a debug action so that it can not get enough memory to fit
+# the null build rows in memory.
select l_orderkey, l_partkey, l_suppkey, l_linenumber
from lineitem
where l_suppkey = 4162 and l_shipmode = 'AIR' and l_returnflag = 'A' and
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test
index 4c0d790..39ae2bb 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test
@@ -21,7 +21,7 @@
# where l_orderkey % 2 = 1 and p_retailprice = l_extendedprice * l_tax
# order by 1,2,3,4 limit 5
#
-set buffer_pool_limit=10m;
+set buffer_pool_limit=90m;
select l_orderkey, l_partkey, l_suppkey, l_linenumber
from lineitem
where if(l_orderkey % 2 = 0, NULL, l_partkey) not in (
@@ -41,7 +41,7 @@ BIGINT,BIGINT,BIGINT,INT
---- QUERY
# Execute NAAJ query 2 in-memory only without enough memory to complete.
set scratch_limit=0;
-set buffer_pool_limit=10m;
+set buffer_pool_limit=90m;
select l_orderkey, l_partkey, l_suppkey, l_linenumber
from lineitem
where if(l_orderkey % 2 = 0, NULL, l_partkey) not in (
@@ -65,7 +65,7 @@ Could not free memory by spilling to disk: scratch_limit is 0
# where p_partkey = 2 and l_partkey % 2 = 1
# order by 1,2,3,4
#
-set buffer_pool_limit=10m;
+set buffer_pool_limit=90m;
select l_orderkey, l_partkey, l_suppkey, l_linenumber
from lineitem
where l_partkey not in (
@@ -86,7 +86,7 @@ BIGINT,BIGINT,BIGINT,INT
# =======================================================================================
# This returns one less row than the previous query because l_partkey=116009 is
# replaced with a NULL.
-set buffer_pool_limit=10m;
+set buffer_pool_limit=90m;
select l_orderkey, l_partkey, l_suppkey, l_linenumber
from lineitem
where if(l_partkey % 5 != 1, NULL, l_partkey) not in (
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
new file mode 100644
index 0000000..f89e537
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
@@ -0,0 +1,66 @@
+# This file contains tests where we don't want the python test framework to supply the
+# debug_action value because the test won't succeed with all possible debug_action values.
+====
+---- QUERY
+# Tests for the case where a spilled partition has 0 probe rows and so we don't build the
+# hash table in a partitioned hash join. Always runs with the minimum reservation to force
+# spilling.
+# INNER JOIN
+set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select straight_join count(*)
+from
+lineitem a, lineitem b
+where
+a.l_partkey = 1 and
+a.l_orderkey = b.l_orderkey;
+---- TYPES
+BIGINT
+---- RESULTS
+173
+---- RUNTIME_PROFILE
+row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# spilled partition with 0 probe rows, NULL AWARE LEFT ANTI JOIN
+set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select straight_join count(*)
+from
+lineitem a
+where
+a.l_partkey not in (select l_partkey from lineitem where l_partkey > 10)
+and a.l_partkey < 1000;
+---- TYPES
+BIGINT
+---- RESULTS
+287
+---- RUNTIME_PROFILE
+row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# spilled partition with 0 probe rows, RIGHT OUTER JOIN
+set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select straight_join count(*)
+from
+supplier right outer join lineitem on s_suppkey = l_suppkey
+where s_acctbal > 0 and s_acctbal < 10;
+---- TYPES
+BIGINT
+---- RESULTS
+12138
+---- RUNTIME_PROFILE
+row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# spilled partition with 0 probe rows, RIGHT ANTI JOIN
+set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
+with x as (select * from supplier limit 10)
+select straight_join count(*)
+from
+x right anti join lineitem on s_suppkey + 100 = l_suppkey;
+---- TYPES
+BIGINT
+---- RESULTS
+5995258
+---- RUNTIME_PROFILE
+row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
index 415fe5d..7d65523 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
@@ -3,7 +3,7 @@
# Test sort with small char column materialized by exprs.
# Set low memory limit to force spilling.
# IMPALA-3332: comparator makes local allocations that cause runaway memory consumption.
-set buffer_pool_limit=4m;
+set buffer_pool_limit=28m;
set mem_limit=200m;
set disable_outermost_topn=1;
select cast(l_comment as char(50))
@@ -41,7 +41,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
---- QUERY
# Test sort with small input char column materialized before sort.
set mem_limit=200m;
-set buffer_pool_limit=4m;
+set buffer_pool_limit=28m;
set disable_outermost_topn=1;
select char_col
from (select cast(l_comment as char(50)) char_col
@@ -80,7 +80,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
# Test sort with large input char column materialized before sort.
# Set low memory limit to force spilling.
set mem_limit=200m;
-set buffer_pool_limit=4m;
+set buffer_pool_limit=28m;
set disable_outermost_topn=1;
select char_col
from (select cast(l_comment as char(200)) char_col
@@ -118,7 +118,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
---- QUERY
# Test sort with varchar column materialized by exprs.
# Set low memory limit to force spilling.
-set buffer_pool_limit=4m;
+set buffer_pool_limit=28m;
# IMPALA-3332: comparator makes local allocations that cause runaway memory consumption.
set mem_limit=200m;
set disable_outermost_topn=1;
@@ -158,7 +158,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
# Test sort with input varchar column materialized before sort.
# Set low memory limit to force spilling.
set mem_limit=200m;
-set buffer_pool_limit=4m;
+set buffer_pool_limit=28m;
set disable_outermost_topn=1;
select char_col
from (select cast(l_comment as varchar(50)) char_col
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/spilling.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling.test b/testdata/workloads/functional-query/queries/QueryTest/spilling.test
index 6639d82..23d9271 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling.test
@@ -1,6 +1,6 @@
====
---- QUERY
-set buffer_pool_limit=15m;
+set buffer_pool_limit=215m;
select count(l1.l_tax)
from
lineitem l1,
@@ -22,7 +22,7 @@ BIGINT
row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
====
---- QUERY
-set buffer_pool_limit=2m;
+set buffer_pool_limit=50m;
select max(t1.total_count), max(t1.l_shipinstruct), max(t1.l_comment) from
(select l_shipinstruct, l_comment, count(*) over () total_count from lineitem) t1
---- RESULTS
@@ -60,7 +60,7 @@ row_regex: .*SpilledPartitions: 0 .*
# Adding TPCH-Q21 in the spilling test to check for IMPALA-1471 (spilling left anti
# and left outer joins were returning wrong results).
# Q21 - Suppliers Who Kept Orders Waiting Query
-set buffer_pool_limit=26m;
+set buffer_pool_limit=200m;
select
s_name,
count(*) as numwait
@@ -211,7 +211,7 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
---- QUERY
# IMPALA-1346/IMPALA-1546: fix sorter memory management so that it can complete
# successfully when in same pipeline as a spilling join.
-set buffer_pool_limit=50m;
+set buffer_pool_limit=170m;
set disable_outermost_topn=1;
select * from lineitem
inner join orders on l_orderkey = o_orderkey
@@ -246,68 +246,6 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
====
---- QUERY
-# Tests for the case where a spilled partition has 0 probe rows and so we don't build the
-# hash table in a partitioned hash join.
-# INNER JOIN
-set buffer_pool_limit=10m;
-select straight_join count(*)
-from
-lineitem a, lineitem b
-where
-a.l_partkey = 1 and
-a.l_orderkey = b.l_orderkey;
----- TYPES
-BIGINT
----- RESULTS
-173
----- RUNTIME_PROFILE
-row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
-====
----- QUERY
-# spilled partition with 0 probe rows, NULL AWARE LEFT ANTI JOIN
-set buffer_pool_limit=10m;
-select straight_join count(*)
-from
-lineitem a
-where
-a.l_partkey not in (select l_partkey from lineitem where l_partkey > 10)
-and a.l_partkey < 1000;
----- TYPES
-BIGINT
----- RESULTS
-287
----- RUNTIME_PROFILE
-row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
-====
----- QUERY
-# spilled partition with 0 probe rows, RIGHT OUTER JOIN
-set buffer_pool_limit=10m;
-select straight_join count(*)
-from
-supplier right outer join lineitem on s_suppkey = l_suppkey
-where s_acctbal > 0 and s_acctbal < 10;
----- TYPES
-BIGINT
----- RESULTS
-12138
----- RUNTIME_PROFILE
-row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
-====
----- QUERY
-# spilled partition with 0 probe rows, RIGHT ANTI JOIN
-set buffer_pool_limit=30m;
-with x as (select * from supplier limit 10)
-select straight_join count(*)
-from
-x right anti join lineitem on s_suppkey + 100 = l_suppkey;
----- TYPES
-BIGINT
----- RESULTS
-5995258
----- RUNTIME_PROFILE
-row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
-====
----- QUERY
# IMPALA-5173: spilling hash join feeding into right side of nested loop join.
# Equivalent to:
# select *
@@ -315,7 +253,7 @@ row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
# where l1.l_quantity = 31.0 and l1.l_tax = 0.03 and l1.l_orderkey <= 100000
# order by l_orderkey, l_partkey, l_suppkey, l_linenumber
# limit 5
-set buffer_pool_limit=9m;
+set buffer_pool_limit=177m;
set num_nodes=1;
select straight_join l.*
from
@@ -345,7 +283,7 @@ bigint,bigint,bigint,int,decimal,decimal,decimal,decimal,string,string,string,st
====
---- QUERY
# Test spilling aggregation when grouping by nondeterministic expression
-set buffer_pool_limit=5m;
+set buffer_pool_limit=79m;
set num_nodes=1;
select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
from tpch_parquet.lineitem
@@ -359,7 +297,7 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
---- QUERY
# Test spilling join with many duplicates in join key. We don't expect this to succeed
# with a memory constraint: see IMPALA-4857.
-set buffer_pool_limit=20m;
+set buffer_pool_limit=167m;
select *
from lineitem l1 join lineitem l2 on l1.l_linenumber = l2.l_linenumber
---- CATCH
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test b/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
index 42f4d50..ceaf1d0 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
@@ -52,14 +52,14 @@ STRING,STRING,BIGINT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING
# Stats are available now.
explain select id from alltypes;
---- RESULTS: VERIFY_IS_EQUAL
-'Max Per-Host Resource Reservation: Memory=0B'
-'Per-Host Resource Estimates: Memory=16.00MB'
+row_regex:.*Max Per-Host Resource Reservation: Memory=.*
+row_regex:.*Per-Host Resource Estimates: Memory=.*
'Codegen disabled by planner'
''
'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-'| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
+row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
'PLAN-ROOT SINK'
-'| mem-estimate=0B mem-reservation=0B'
+row_regex:.*mem-estimate=.* mem-reservation=.*
'|'
'00:SCAN HDFS [$DATABASE.alltypes]'
row_regex:.*partitions=12/12 files=12 size=.*
@@ -77,14 +77,14 @@ STRING
# Select a subset of partitions.
explain select id from alltypes where month in (1, 2, 3);
---- RESULTS: VERIFY_IS_EQUAL
-'Max Per-Host Resource Reservation: Memory=0B'
-'Per-Host Resource Estimates: Memory=16.00MB'
+row_regex:.*Max Per-Host Resource Reservation: Memory=.*
+row_regex:.*Per-Host Resource Estimates: Memory=.*
'Codegen disabled by planner'
''
'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-'| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
+row_regex:.*mem-estimate=.* mem-reservation=.*
'PLAN-ROOT SINK'
-'| mem-estimate=0B mem-reservation=0B'
+row_regex:.*mem-estimate=.* mem-reservation=.*
'|'
'00:SCAN HDFS [$DATABASE.alltypes]'
row_regex:.*partitions=3/12 files=3 size=.*
@@ -104,13 +104,13 @@ insert into alltypes partition(year, month)
select * from functional_parquet.alltypes where year = 2009;
explain select id from alltypes;
---- RESULTS: VERIFY_IS_EQUAL
-'Max Per-Host Resource Reservation: Memory=0B'
-'Per-Host Resource Estimates: Memory=16.00MB'
+row_regex:.*Max Per-Host Resource Reservation: Memory=.*
+row_regex:.*Per-Host Resource Estimates: Memory=.*
''
'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-'| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
+row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
'PLAN-ROOT SINK'
-'| mem-estimate=0B mem-reservation=0B'
+row_regex:.*mem-estimate=.* mem-reservation=.*
'|'
'00:SCAN HDFS [$DATABASE.alltypes]'
row_regex:.*partitions=12/12 files=24 size=.*
@@ -130,14 +130,14 @@ insert into alltypes partition(year, month)
select * from functional_parquet.alltypes where year = 2010;
explain select id from alltypes where year = 2010;
---- RESULTS: VERIFY_IS_EQUAL
-'Max Per-Host Resource Reservation: Memory=0B'
-'Per-Host Resource Estimates: Memory=16.00MB'
+row_regex:.*Max Per-Host Resource Reservation: Memory=.*
+row_regex:.*Per-Host Resource Estimates: Memory=16.00MB'
'Codegen disabled by planner'
''
'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-'| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
+row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
'PLAN-ROOT SINK'
-'| mem-estimate=0B mem-reservation=0B'
+row_regex:.*mem-estimate=.* mem-reservation=.*
'|'
'00:SCAN HDFS [$DATABASE.alltypes]'
row_regex:.*partitions=12/24 files=12 size=.*
@@ -156,14 +156,14 @@ STRING
compute stats alltypes;
explain select id from alltypes where year = 2010;
---- RESULTS: VERIFY_IS_EQUAL
-'Max Per-Host Resource Reservation: Memory=0B'
-'Per-Host Resource Estimates: Memory=16.00MB'
+row_regex:.*Max Per-Host Resource Reservation: Memory=.*
+row_regex:.*Per-Host Resource Estimates: Memory=16.00MB'
'Codegen disabled by planner'
''
'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-'| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
+row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
'PLAN-ROOT SINK'
-'| mem-estimate=0B mem-reservation=0B'
+row_regex:.*mem-estimate=.* mem-reservation=.*
'|'
'00:SCAN HDFS [$DATABASE.alltypes]'
row_regex:.*partitions=12/24 files=12 size=.*
http://git-wip-us.apache.org/repos/asf/impala/blob/fb5dc9eb/testdata/workloads/tpch/queries/sort-reservation-usage.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/tpch/queries/sort-reservation-usage.test b/testdata/workloads/tpch/queries/sort-reservation-usage.test
index 92f180d..af31ccb 100644
--- a/testdata/workloads/tpch/queries/sort-reservation-usage.test
+++ b/testdata/workloads/tpch/queries/sort-reservation-usage.test
@@ -1,17 +1,18 @@
====
---- QUERY
# Test that in-mem sorts incrementally give up memory when emitting output.
-# This query and the limit is calibrated to fail if the first sort does not
-# give up memory to the second sort.
+# This query and scratch limit are chosen so that the query fails if the first sort
+# does not give up memory to the second sort. Scans the text formatted file so that
+# the scan uses less reservation.
set num_nodes=1;
set scratch_limit=0;
-set buffer_pool_limit=15m;
+set buffer_pool_limit=35m;
set default_spillable_buffer_size=64kb;
SELECT *
FROM (SELECT
Rank() OVER(ORDER BY l_orderkey) AS rank,
Rank() OVER(ORDER BY l_partkey) AS rank2
- FROM lineitem
+ FROM tpch.lineitem
WHERE l_shipdate < '1992-05-09') a
WHERE rank < 10
ORDER BY rank;