You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/08/23 03:29:00 UTC
[3/4] incubator-impala git commit: IMPALA-3208: max_row_size option
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
index 73129ec..bf2bc52 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
@@ -415,7 +415,7 @@ Per-Host Resources: mem-estimate=106.24MB mem-reservation=0B
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=8.50MB
+Max Per-Host Resource Reservation: Memory=9.50MB
Per-Host Resource Estimates: Memory=232.48MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -428,11 +428,11 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=16B cardinality=1563438
|
F01:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
-Per-Host Resources: mem-estimate=20.00MB mem-reservation=8.50MB
+Per-Host Resources: mem-estimate=20.00MB mem-reservation=9.50MB
03:AGGREGATE [FINALIZE]
| output: count:merge(*)
| group by: l_orderkey
-| mem-estimate=10.00MB mem-reservation=4.25MB spill-buffer=256.00KB
+| mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB
| tuple-ids=1 row-size=16B cardinality=1563438
|
02:EXCHANGE [HASH(l_orderkey)]
@@ -1548,8 +1548,8 @@ PLAN-ROOT SINK
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=78B cardinality=600122
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=38.25MB
-Per-Host Resource Estimates: Memory=343.83MB
+Max Per-Host Resource Reservation: Memory=38.75MB
+Per-Host Resource Estimates: Memory=344.33MB
F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -1643,7 +1643,7 @@ Per-Host Resources: mem-estimate=97.00MB mem-reservation=34.00MB
| tuple-ids=2 row-size=70B cardinality=575772
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
-Per-Host Resources: mem-estimate=46.83MB mem-reservation=4.25MB
+Per-Host Resources: mem-estimate=47.33MB mem-reservation=4.75MB
04:AGGREGATE [STREAMING]
| group by: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
| mem-estimate=42.58MB mem-reservation=0B spill-buffer=2.00MB
@@ -1653,7 +1653,7 @@ Per-Host Resources: mem-estimate=46.83MB mem-reservation=4.25MB
| hash predicates: l_orderkey = o_orderkey
| fk/pk conjuncts: l_orderkey = o_orderkey
| runtime filters: RF000 <- o_orderkey
-| mem-estimate=4.25MB mem-reservation=4.25MB spill-buffer=256.00KB
+| mem-estimate=4.75MB mem-reservation=4.75MB spill-buffer=256.00KB
| tuple-ids=0,1 row-size=86B cardinality=575772
|
|--12:EXCHANGE [HASH(o_orderkey)]
@@ -1688,8 +1688,8 @@ Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=78B cardinality=600122
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=72.25MB
-Per-Host Resource Estimates: Memory=683.41MB
+Max Per-Host Resource Reservation: Memory=73.75MB
+Per-Host Resource Estimates: Memory=684.91MB
F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -1799,7 +1799,7 @@ Per-Host Resources: mem-estimate=194.00MB mem-reservation=68.00MB
| tuple-ids=2 row-size=70B cardinality=575772
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
-Per-Host Resources: mem-estimate=89.41MB mem-reservation=4.25MB
+Per-Host Resources: mem-estimate=90.91MB mem-reservation=5.75MB
04:AGGREGATE [STREAMING]
| group by: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
| mem-estimate=42.58MB mem-reservation=0B spill-buffer=2.00MB
@@ -1810,7 +1810,7 @@ Per-Host Resources: mem-estimate=89.41MB mem-reservation=4.25MB
| hash predicates: l_orderkey = o_orderkey
| fk/pk conjuncts: l_orderkey = o_orderkey
| runtime filters: RF000 <- o_orderkey
-| mem-estimate=2.12MB mem-reservation=2.12MB spill-buffer=128.00KB
+| mem-estimate=2.88MB mem-reservation=2.88MB spill-buffer=128.00KB
| tuple-ids=0,1 row-size=86B cardinality=575772
|
|--F12:PLAN FRAGMENT [HASH(l_orderkey)] hosts=2 instances=4
@@ -1888,11 +1888,11 @@ order by
o_orderdate
limit 100
---- PLAN
-Max Per-Host Resource Reservation: Memory=80.75MB
+Max Per-Host Resource Reservation: Memory=81.25MB
Per-Host Resource Estimates: Memory=393.18MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=393.18MB mem-reservation=80.75MB
+| Per-Host Resources: mem-estimate=393.18MB mem-reservation=81.25MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -1910,14 +1910,14 @@ PLAN-ROOT SINK
07:HASH JOIN [LEFT SEMI JOIN]
| hash predicates: o_orderkey = l_orderkey
| runtime filters: RF000 <- l_orderkey
-| mem-estimate=4.25MB mem-reservation=4.25MB spill-buffer=256.00KB
+| mem-estimate=4.75MB mem-reservation=4.75MB spill-buffer=256.00KB
| tuple-ids=2,1,0 row-size=108B cardinality=575772
|
|--04:AGGREGATE [FINALIZE]
| | output: sum(l_quantity)
| | group by: l_orderkey
| | having: sum(l_quantity) > 300
-| | mem-estimate=10.00MB mem-reservation=4.25MB spill-buffer=256.00KB
+| | mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB
| | tuple-ids=4 row-size=24B cardinality=156344
| |
| 03:SCAN HDFS [tpch.lineitem]
@@ -1968,8 +1968,8 @@ PLAN-ROOT SINK
mem-estimate=88.00MB mem-reservation=0B
tuple-ids=2 row-size=16B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=82.88MB
-Per-Host Resource Estimates: Memory=510.80MB
+Max Per-Host Resource Reservation: Memory=84.12MB
+Per-Host Resource Estimates: Memory=511.55MB
F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -2000,7 +2000,7 @@ Per-Host Resources: mem-estimate=60.41MB mem-reservation=34.00MB
| tuple-ids=6 row-size=100B cardinality=575772
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
-Per-Host Resources: mem-estimate=115.03MB mem-reservation=48.88MB
+Per-Host Resources: mem-estimate=115.78MB mem-reservation=50.12MB
08:AGGREGATE [STREAMING]
| output: sum(l_quantity)
| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
@@ -2010,14 +2010,14 @@ Per-Host Resources: mem-estimate=115.03MB mem-reservation=48.88MB
07:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]
| hash predicates: o_orderkey = l_orderkey
| runtime filters: RF000 <- l_orderkey
-| mem-estimate=2.12MB mem-reservation=2.12MB spill-buffer=128.00KB
+| mem-estimate=2.88MB mem-reservation=2.88MB spill-buffer=128.00KB
| tuple-ids=2,1,0 row-size=108B cardinality=575772
|
|--14:AGGREGATE [FINALIZE]
| | output: sum:merge(l_quantity)
| | group by: l_orderkey
| | having: sum(l_quantity) > 300
-| | mem-estimate=10.00MB mem-reservation=4.25MB spill-buffer=256.00KB
+| | mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB
| | tuple-ids=4 row-size=24B cardinality=156344
| |
| 13:EXCHANGE [HASH(l_orderkey)]
@@ -2098,8 +2098,8 @@ Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
mem-estimate=88.00MB mem-reservation=0B
tuple-ids=2 row-size=16B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=121.12MB
-Per-Host Resource Estimates: Memory=965.47MB
+Max Per-Host Resource Reservation: Memory=122.88MB
+Per-Host Resource Estimates: Memory=967.22MB
F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -2130,7 +2130,7 @@ Per-Host Resources: mem-estimate=120.82MB mem-reservation=68.00MB
| tuple-ids=6 row-size=100B cardinality=575772
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
-Per-Host Resources: mem-estimate=173.93MB mem-reservation=53.12MB
+Per-Host Resources: mem-estimate=175.68MB mem-reservation=54.88MB
08:AGGREGATE [STREAMING]
| output: sum(l_quantity)
| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
@@ -2141,7 +2141,7 @@ Per-Host Resources: mem-estimate=173.93MB mem-reservation=53.12MB
| hash-table-id=00
| hash predicates: o_orderkey = l_orderkey
| runtime filters: RF000 <- l_orderkey
-| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB
+| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB
| tuple-ids=2,1,0 row-size=108B cardinality=575772
|
|--F08:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
@@ -2155,7 +2155,7 @@ Per-Host Resources: mem-estimate=173.93MB mem-reservation=53.12MB
| | output: sum:merge(l_quantity)
| | group by: l_orderkey
| | having: sum(l_quantity) > 300
-| | mem-estimate=10.00MB mem-reservation=4.25MB spill-buffer=256.00KB
+| | mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB
| | tuple-ids=4 row-size=24B cardinality=156344
| |
| 13:EXCHANGE [HASH(l_orderkey)]
@@ -2390,13 +2390,13 @@ from tpch_nested_parquet.customer c,
join c.c_orders o2 on o1.o_orderkey = o2.o_orderkey
order by o1.o_orderkey limit 100) v
---- PLAN
-Max Per-Host Resource Reservation: Memory=69.06MB
-Per-Host Resource Estimates: Memory=345.06MB
+Max Per-Host Resource Reservation: Memory=69.94MB
+Per-Host Resource Estimates: Memory=345.94MB
WARNING: The following tables are missing relevant table and/or column statistics.
tpch_nested_parquet.customer
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=345.06MB mem-reservation=69.06MB
+| Per-Host Resources: mem-estimate=345.94MB mem-reservation=69.94MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -2431,7 +2431,7 @@ PLAN-ROOT SINK
| 05:HASH JOIN [INNER JOIN]
| | hash predicates: o1.o_orderkey = o2.o_orderkey
| | fk/pk conjuncts: assumed fk/pk
-| | mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB
| | tuple-ids=1,2 row-size=32B cardinality=10
| |
| |--04:UNNEST [c.c_orders o2]
@@ -2452,8 +2452,8 @@ PLAN-ROOT SINK
mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=66B cardinality=150000
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=69.06MB
-Per-Host Resource Estimates: Memory=473.06MB
+Max Per-Host Resource Reservation: Memory=69.94MB
+Per-Host Resource Estimates: Memory=473.94MB
WARNING: The following tables are missing relevant table and/or column statistics.
tpch_nested_parquet.customer
@@ -2478,7 +2478,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=34.00MB
| tuple-ids=6 row-size=58B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=345.06MB mem-reservation=35.06MB
+Per-Host Resources: mem-estimate=345.94MB mem-reservation=35.94MB
09:AGGREGATE [STREAMING]
| group by: c_name, o1.o_orderkey, o2.o_orderstatus
| mem-estimate=128.00MB mem-reservation=0B spill-buffer=2.00MB
@@ -2510,7 +2510,7 @@ Per-Host Resources: mem-estimate=345.06MB mem-reservation=35.06MB
| 05:HASH JOIN [INNER JOIN]
| | hash predicates: o1.o_orderkey = o2.o_orderkey
| | fk/pk conjuncts: assumed fk/pk
-| | mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB
| | tuple-ids=1,2 row-size=32B cardinality=10
| |
| |--04:UNNEST [c.c_orders o2]
@@ -2531,8 +2531,8 @@ Per-Host Resources: mem-estimate=345.06MB mem-reservation=35.06MB
mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=66B cardinality=150000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=138.12MB
-Per-Host Resource Estimates: Memory=946.13MB
+Max Per-Host Resource Reservation: Memory=139.88MB
+Per-Host Resource Estimates: Memory=947.88MB
WARNING: The following tables are missing relevant table and/or column statistics.
tpch_nested_parquet.customer
@@ -2557,7 +2557,7 @@ Per-Host Resources: mem-estimate=256.00MB mem-reservation=68.00MB
| tuple-ids=6 row-size=58B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=690.13MB mem-reservation=70.12MB
+Per-Host Resources: mem-estimate=691.88MB mem-reservation=71.88MB
09:AGGREGATE [STREAMING]
| group by: c_name, o1.o_orderkey, o2.o_orderstatus
| mem-estimate=128.00MB mem-reservation=0B spill-buffer=2.00MB
@@ -2589,7 +2589,7 @@ Per-Host Resources: mem-estimate=690.13MB mem-reservation=70.12MB
| 05:HASH JOIN [INNER JOIN]
| | hash predicates: o1.o_orderkey = o2.o_orderkey
| | fk/pk conjuncts: assumed fk/pk
-| | mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB
| | tuple-ids=1,2 row-size=32B cardinality=10
| |
| |--04:UNNEST [c.c_orders o2]
@@ -3019,8 +3019,8 @@ Per-Host Resources: mem-estimate=77.77MB mem-reservation=34.00MB
mem-estimate=40.00MB mem-reservation=0B
tuple-ids=0 row-size=191B cardinality=1500000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=93.50MB
-Per-Host Resource Estimates: Memory=421.03MB
+Max Per-Host Resource Reservation: Memory=94.50MB
+Per-Host Resource Estimates: Memory=422.03MB
F05:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -3053,7 +3053,7 @@ Per-Host Resources: mem-estimate=155.53MB mem-reservation=68.00MB
| | tuple-ids=1,2,3 row-size=24B cardinality=1500000
| |
| F04:PLAN FRAGMENT [HASH(t3.o_orderkey)] hosts=2 instances=4
-| Per-Host Resources: mem-estimate=25.50MB mem-reservation=25.50MB
+| Per-Host Resources: mem-estimate=26.50MB mem-reservation=26.50MB
| 05:HASH JOIN [INNER JOIN, PARTITIONED]
| | hash-table-id=01
| | hash predicates: t2.o_orderkey = t3.o_orderkey
@@ -3074,7 +3074,7 @@ Per-Host Resources: mem-estimate=155.53MB mem-reservation=68.00MB
| | | hash predicates: t3.o_orderkey = t4.o_orderkey
| | | fk/pk conjuncts: t3.o_orderkey = t4.o_orderkey
| | | runtime filters: RF002 <- t4.o_orderkey
-| | | mem-estimate=4.25MB mem-reservation=4.25MB spill-buffer=256.00KB
+| | | mem-estimate=4.75MB mem-reservation=4.75MB spill-buffer=256.00KB
| | | tuple-ids=2,3 row-size=16B cardinality=1500000
| | |
| | |--F08:PLAN FRAGMENT [HASH(t3.o_orderkey)] hosts=2 instances=4
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
index 16e8cb2..340c3ec 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
@@ -3,8 +3,8 @@ select straight_join *
from tpch_parquet.customer
inner join tpch_parquet.nation on c_nationkey = n_nationkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=1.06MB
-Per-Host Resource Estimates: Memory=41.06MB
+Max Per-Host Resource Reservation: Memory=1.94MB
+Per-Host Resource Estimates: Memory=41.94MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -16,12 +16,12 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=355B cardinality=150000
|
F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=25.06MB mem-reservation=1.06MB
+Per-Host Resources: mem-estimate=25.94MB mem-reservation=1.94MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash predicates: c_nationkey = n_nationkey
| fk/pk conjuncts: c_nationkey = n_nationkey
| runtime filters: RF000 <- n_nationkey
-| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB
+| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB
| tuple-ids=0,1 row-size=355B cardinality=150000
|
|--03:EXCHANGE [BROADCAST]
@@ -47,8 +47,8 @@ Per-Host Resources: mem-estimate=25.06MB mem-reservation=1.06MB
mem-estimate=24.00MB mem-reservation=0B
tuple-ids=0 row-size=238B cardinality=150000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=2.12MB
-Per-Host Resource Estimates: Memory=82.12MB
+Max Per-Host Resource Reservation: Memory=3.88MB
+Per-Host Resource Estimates: Memory=83.88MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -60,13 +60,13 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=355B cardinality=150000
|
F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-Per-Host Resources: mem-estimate=50.12MB mem-reservation=2.12MB
+Per-Host Resources: mem-estimate=51.88MB mem-reservation=3.88MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash-table-id=00
| hash predicates: c_nationkey = n_nationkey
| fk/pk conjuncts: c_nationkey = n_nationkey
| runtime filters: RF000 <- n_nationkey
-| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB
+| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB
| tuple-ids=0,1 row-size=355B cardinality=150000
|
|--F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
@@ -516,7 +516,7 @@ select c_nationkey, avg(c_acctbal)
from tpch_parquet.customer
group by c_nationkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=1.06MB
+Max Per-Host Resource Reservation: Memory=1.94MB
Per-Host Resource Estimates: Memory=44.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -529,11 +529,11 @@ PLAN-ROOT SINK
| tuple-ids=2 row-size=10B cardinality=25
|
F01:PLAN FRAGMENT [HASH(c_nationkey)] hosts=1 instances=1
-Per-Host Resources: mem-estimate=10.00MB mem-reservation=1.06MB
+Per-Host Resources: mem-estimate=10.00MB mem-reservation=1.94MB
03:AGGREGATE [FINALIZE]
| output: avg:merge(c_acctbal)
| group by: c_nationkey
-| mem-estimate=10.00MB mem-reservation=1.06MB spill-buffer=64.00KB
+| mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB
| tuple-ids=2 row-size=10B cardinality=25
|
02:EXCHANGE [HASH(c_nationkey)]
@@ -556,7 +556,7 @@ Per-Host Resources: mem-estimate=34.00MB mem-reservation=0B
mem-estimate=24.00MB mem-reservation=0B
tuple-ids=0 row-size=10B cardinality=150000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=2.12MB
+Max Per-Host Resource Reservation: Memory=3.88MB
Per-Host Resource Estimates: Memory=88.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -569,11 +569,11 @@ PLAN-ROOT SINK
| tuple-ids=2 row-size=10B cardinality=25
|
F01:PLAN FRAGMENT [HASH(c_nationkey)] hosts=1 instances=2
-Per-Host Resources: mem-estimate=20.00MB mem-reservation=2.12MB
+Per-Host Resources: mem-estimate=20.00MB mem-reservation=3.88MB
03:AGGREGATE [FINALIZE]
| output: avg:merge(c_acctbal)
| group by: c_nationkey
-| mem-estimate=10.00MB mem-reservation=1.06MB spill-buffer=64.00KB
+| mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB
| tuple-ids=2 row-size=10B cardinality=25
|
02:EXCHANGE [HASH(c_nationkey)]
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
index 7e78f73..a4c06e0 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
@@ -154,14 +154,14 @@ select id from functional.alltypes t1 where exists (
where t1.id = t2.id)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=161.06MB mem-reservation=1.06MB
+| Per-Host Resources: mem-estimate=161.94MB mem-reservation=1.94MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
02:HASH JOIN [LEFT SEMI JOIN]
| hash predicates: t1.id = t2.id
| runtime filters: RF000 <- t2.id
-| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB
+| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB
| tuple-ids=0 row-size=4B cardinality=10
|
|--01:SCAN HDFS [functional.alltypessmall t2]
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/testdata/workloads/functional-query/queries/QueryTest/set.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/set.test b/testdata/workloads/functional-query/queries/QueryTest/set.test
index 12b8df6..a7004e0 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/set.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/set.test
@@ -260,3 +260,34 @@ explain select count(distinct double_col) from functional.alltypesagg;
# IMPALA-5591: This shouldn't throw an error.
set scratch_limit=-1;
====
+---- QUERY
+# Power of two max_row_size.
+set max_row_size=8m;
+set;
+---- RESULTS: VERIFY_IS_SUBSET
+'MAX_ROW_SIZE','8388608'
+---- TYPES
+STRING,STRING
+====
+---- QUERY
+# Non power of two max_row_size.
+set max_row_size=12345;
+set;
+---- RESULTS: VERIFY_IS_SUBSET
+'MAX_ROW_SIZE','12345'
+---- TYPES
+STRING,STRING
+====
+---- QUERY
+set max_row_size=8m;
+====
+---- QUERY
+set max_row_size=-1;
+---- CATCH
+Max row size must be a positive number of bytes: -1
+====
+---- QUERY
+set max_row_size=0;
+---- CATCH
+Max row size must be a positive number of bytes: 0
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
index 34628c0..b7345f2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
@@ -169,6 +169,7 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
---- QUERY
# Test aggregation with minimum required reservation to exercise IMPALA-2708.
# Merge aggregation requires 17 buffers. The buffer size is 256k for this test.
+set max_row_size=256k;
set buffer_pool_limit=4352k;
select count(*)
from (select distinct * from orders) t
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/testdata/workloads/functional-query/queries/QueryTest/spilling-large-rows.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-large-rows.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-large-rows.test
new file mode 100644
index 0000000..9b61986
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-large-rows.test
@@ -0,0 +1,188 @@
+====
+---- QUERY
+# Create a temporary table with 10MB strings for the following tests.
+create table bigstrs stored as parquet as
+select *, repeat(string_col, 10000000) as bigstr
+from functional.alltypes
+order by id
+limit 25
+---- RESULTS
+'Inserted 25 row(s)'
+====
+---- QUERY
+# Row is too big to process in agg.
+select id, count(distinct bigstr)
+from bigstrs
+group by id
+---- CATCH
+Row of size 9.54 MB could not be materialized in plan node
+====
+---- QUERY
+# Agg should be able to process the large strings if we increase the row size.
+set max_row_size=10m;
+select id, count(distinct bigstr)
+from bigstrs
+group by id
+order by id
+---- TYPES
+int,bigint
+---- RESULTS
+0,1
+1,1
+2,1
+3,1
+4,1
+5,1
+6,1
+7,1
+8,1
+9,1
+10,1
+11,1
+12,1
+13,1
+14,1
+15,1
+16,1
+17,1
+18,1
+19,1
+20,1
+21,1
+22,1
+23,1
+24,1
+====
+---- QUERY
+# Row is too big to process in right side of hash join.
+select straight_join at.id, bs.id, at.string_col
+from functional.alltypes at
+ join bigstrs bs on repeat(at.string_col, 10000) = substring(bs.bigstr, 5000000, 10000) and at.id = bs.id
+where at.id < 100
+---- CATCH
+Row of size 9.54 MB could not be materialized in plan node with id 2. Increase the max_row_size query option (currently 512.00 KB) to process larger rows.
+====
+---- QUERY
+# Row is too big to process in right side of hash join.
+set max_row_size=18m;
+select straight_join at.id, bs.id, at.string_col
+from functional.alltypes at
+ join bigstrs bs on repeat(at.string_col, 10000) = substring(bs.bigstr, 5000000, 10000) and at.id = bs.id
+where at.id < 100
+---- TYPES
+int,int,string
+---- RESULTS
+0,0,'0'
+1,1,'1'
+2,2,'2'
+3,3,'3'
+4,4,'4'
+5,5,'5'
+6,6,'6'
+7,7,'7'
+8,8,'8'
+9,9,'9'
+10,10,'0'
+11,11,'1'
+12,12,'2'
+13,13,'3'
+14,14,'4'
+15,15,'5'
+16,16,'6'
+17,17,'7'
+18,18,'8'
+19,19,'9'
+20,20,'0'
+21,21,'1'
+22,22,'2'
+23,23,'3'
+24,24,'4'
+====
+---- QUERY
+# Row is too big to process in sort.
+select id, substr(bigstr, 1, 5)
+from bigstrs
+order by bigstr, id
+---- CATCH
+Row of size 9.54 MB could not be materialized in plan node with id 1. Increase the max_row_size query option (currently 512.00 KB) to process larger rows.
+====
+---- QUERY
+# Sort should be able to process the large strings if we increase the row size.
+set max_row_size=10m;
+select id, substr(bigstr, 1, 5)
+from bigstrs
+where id < 15
+order by bigstr, id
+---- TYPES
+int,string
+---- RESULTS
+0,'00000'
+10,'00000'
+1,'11111'
+11,'11111'
+2,'22222'
+12,'22222'
+3,'33333'
+13,'33333'
+4,'44444'
+14,'44444'
+5,'55555'
+6,'66666'
+7,'77777'
+8,'88888'
+9,'99999'
+====
+---- QUERY
+# Row is too big to process in sort or analytic.
+SELECT id, int_col, substring(bigstr, 1, 10), substring(bigstr, 9999999, 1), rank
+FROM (
+ SELECT id, int_col, bigstr, Rank() OVER (
+ ORDER BY int_col
+ ) AS rank
+ FROM bigstrs
+ ) a
+ORDER BY id
+---- CATCH
+Row of size 9.54 MB could not be materialized in plan node with id 1. Increase the max_row_size query option (currently 512.00 KB) to process larger rows.
+====
+---- QUERY
+# Sort and analytic should be able to process the large strings if we increase the row
+# size.
+set max_row_size=10m;
+SELECT id, int_col, substring(bigstr, 1, 10), substring(bigstr, 9999999, 1), rank
+FROM (
+ SELECT id, int_col, bigstr, Rank() OVER (
+ ORDER BY int_col
+ ) AS rank
+ FROM bigstrs
+ ) a
+ORDER BY id
+---- TYPES
+int,int,string,string,bigint
+---- RESULTS
+0,0,'0000000000','0',1
+1,1,'1111111111','1',4
+2,2,'2222222222','2',7
+3,3,'3333333333','3',10
+4,4,'4444444444','4',13
+5,5,'5555555555','5',16
+6,6,'6666666666','6',18
+7,7,'7777777777','7',20
+8,8,'8888888888','8',22
+9,9,'9999999999','9',24
+10,0,'0000000000','0',1
+11,1,'1111111111','1',4
+12,2,'2222222222','2',7
+13,3,'3333333333','3',10
+14,4,'4444444444','4',13
+15,5,'5555555555','5',16
+16,6,'6666666666','6',18
+17,7,'7777777777','7',20
+18,8,'8888888888','8',22
+19,9,'9999999999','9',24
+20,0,'0000000000','0',1
+21,1,'1111111111','1',4
+22,2,'2222222222','2',7
+23,3,'3333333333','3',10
+24,4,'4444444444','4',13
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/tests/query_test/test_spilling.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py
index ee366d4..0ca0971 100644
--- a/tests/query_test/test_spilling.py
+++ b/tests/query_test/test_spilling.py
@@ -52,6 +52,11 @@ class TestSpilling(ImpalaTestSuite):
def test_spilling_aggs(self, vector):
self.run_test_case('QueryTest/spilling-aggs', vector)
+ def test_spilling_large_rows(self, vector, unique_database):
+ """Test that we can process large rows in spilling operators, with or without
+ spilling to disk"""
+ self.run_test_case('QueryTest/spilling-large-rows', vector, unique_database)
+
def test_spilling_sorts_exhaustive(self, vector):
if self.exploration_strategy() != 'exhaustive':
pytest.skip("only run large sorts on exhaustive")