You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/04/19 01:20:31 UTC
[4/6] incubator-impala git commit: IMPALA-3748: minimum buffer
requirements in planner
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
new file mode 100644
index 0000000..7ae30fa
--- /dev/null
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
@@ -0,0 +1,1121 @@
+# Parquet scan
+select * from tpch_parquet.lineitem
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=80.00MB
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+01:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0 row-size=263B cardinality=6001215
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
+ partitions=1/1 files=3 size=193.61MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=80.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=160.00MB
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+01:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0 row-size=263B cardinality=6001215
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
+ partitions=1/1 files=3 size=193.61MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=80.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+====
+# Text scan
+select * from tpch.lineitem;
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=88.00MB
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+01:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0 row-size=263B cardinality=6001215
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+00:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=176.00MB
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+01:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0 row-size=263B cardinality=6001215
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+00:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+====
+# HBase scan
+select * from functional_hbase.alltypes
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=1.00GB
+WARNING: The following tables are missing relevant table and/or column statistics.
+functional_hbase.alltypes
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+01:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0 row-size=88B cardinality=14298
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=100 instances=100
+00:SCAN HBASE [functional_hbase.alltypes]
+ table stats: unavailable
+ column stats: unavailable
+ mem-estimate=1.00GB mem-reservation=0B
+ tuple-ids=0 row-size=88B cardinality=14298
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=2.00GB
+WARNING: The following tables are missing relevant table and/or column statistics.
+functional_hbase.alltypes
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+01:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0 row-size=88B cardinality=14298
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=100 instances=200
+00:SCAN HBASE [functional_hbase.alltypes]
+ table stats: unavailable
+ column stats: unavailable
+ mem-estimate=1.00GB mem-reservation=0B
+ tuple-ids=0 row-size=88B cardinality=14298
+====
+# Data source scan
+select * from functional.alltypes_datasource
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=1.00GB
+WARNING: The following tables are missing relevant table and/or column statistics.
+functional.alltypes_datasource
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+01:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0 row-size=116B cardinality=5000
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+00:SCAN DATA SOURCE [functional.alltypes_datasource]
+ mem-estimate=1.00GB mem-reservation=0B
+ tuple-ids=0 row-size=116B cardinality=5000
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=2.00GB
+WARNING: The following tables are missing relevant table and/or column statistics.
+functional.alltypes_datasource
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+01:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0 row-size=116B cardinality=5000
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
+00:SCAN DATA SOURCE [functional.alltypes_datasource]
+ mem-estimate=1.00GB mem-reservation=0B
+ tuple-ids=0 row-size=116B cardinality=5000
+====
+# Union
+select * from tpch.lineitem
+union all
+select * from tpch.lineitem
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=176.00MB
+
+F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+03:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=2 row-size=263B cardinality=12002430
+|
+F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+00:UNION
+| pass-through-operands: all
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=2 row-size=263B cardinality=12002430
+|
+|--02:SCAN HDFS [tpch.lineitem, RANDOM]
+| partitions=1/1 files=1 size=718.94MB
+| table stats: 6001215 rows total
+| column stats: all
+| mem-estimate=88.00MB mem-reservation=0B
+| tuple-ids=1 row-size=263B cardinality=6001215
+|
+01:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=352.00MB
+
+F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+03:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=2 row-size=263B cardinality=12002430
+|
+F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+00:UNION
+| pass-through-operands: all
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=2 row-size=263B cardinality=12002430
+|
+|--02:SCAN HDFS [tpch.lineitem, RANDOM]
+| partitions=1/1 files=1 size=718.94MB
+| table stats: 6001215 rows total
+| column stats: all
+| mem-estimate=88.00MB mem-reservation=0B
+| tuple-ids=1 row-size=263B cardinality=6001215
+|
+01:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+====
+# Grouping aggregation
+select l_orderkey, count(*)
+from tpch_parquet.lineitem
+group by l_orderkey
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=264.00MB
+Per-Host Resource Estimates: Memory=116.24MB
+
+F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+04:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=1 row-size=16B cardinality=1563438
+|
+F01:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
+03:AGGREGATE [FINALIZE]
+| output: count:merge(*)
+| group by: l_orderkey
+| mem-estimate=10.00MB mem-reservation=264.00MB
+| tuple-ids=1 row-size=16B cardinality=1563438
+|
+02:EXCHANGE [HASH(l_orderkey)]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=1 row-size=16B cardinality=1563438
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+01:AGGREGATE [STREAMING]
+| output: count(*)
+| group by: l_orderkey
+| mem-estimate=26.24MB mem-reservation=0B
+| tuple-ids=1 row-size=16B cardinality=1563438
+|
+00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
+ partitions=1/1 files=3 size=193.61MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=80.00MB mem-reservation=0B
+ tuple-ids=0 row-size=8B cardinality=6001215
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=528.00MB
+Per-Host Resource Estimates: Memory=232.48MB
+
+F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+04:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=1 row-size=16B cardinality=1563438
+|
+F01:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
+03:AGGREGATE [FINALIZE]
+| output: count:merge(*)
+| group by: l_orderkey
+| mem-estimate=10.00MB mem-reservation=264.00MB
+| tuple-ids=1 row-size=16B cardinality=1563438
+|
+02:EXCHANGE [HASH(l_orderkey)]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=1 row-size=16B cardinality=1563438
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+01:AGGREGATE [STREAMING]
+| output: count(*)
+| group by: l_orderkey
+| mem-estimate=26.24MB mem-reservation=0B
+| tuple-ids=1 row-size=16B cardinality=1563438
+|
+00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
+ partitions=1/1 files=3 size=193.61MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=80.00MB mem-reservation=0B
+ tuple-ids=0 row-size=8B cardinality=6001215
+====
+# Non-grouping aggregation with zero-slot parquet scan
+select count(*) from tpch_parquet.lineitem
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=20.00MB
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+03:AGGREGATE [FINALIZE]
+| output: count:merge(*)
+| mem-estimate=10.00MB mem-reservation=0B
+| tuple-ids=1 row-size=8B cardinality=1
+|
+02:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=1 row-size=8B cardinality=1
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+01:AGGREGATE
+| output: count(*)
+| mem-estimate=10.00MB mem-reservation=0B
+| tuple-ids=1 row-size=8B cardinality=1
+|
+00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
+ partitions=1/1 files=3 size=193.61MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=0B mem-reservation=0B
+ tuple-ids=0 row-size=0B cardinality=6001215
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=180.00MB
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+03:AGGREGATE [FINALIZE]
+| output: count:merge(*)
+| mem-estimate=10.00MB mem-reservation=0B
+| tuple-ids=1 row-size=8B cardinality=1
+|
+02:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=1 row-size=8B cardinality=1
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+01:AGGREGATE
+| output: count(*)
+| mem-estimate=10.00MB mem-reservation=0B
+| tuple-ids=1 row-size=8B cardinality=1
+|
+00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
+ partitions=1/1 files=3 size=193.61MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=80.00MB mem-reservation=0B
+ tuple-ids=0 row-size=0B cardinality=6001215
+====
+# Sort
+select *
+from tpch_parquet.lineitem
+order by l_comment
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=48.00MB
+Per-Host Resource Estimates: Memory=240.00MB
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+02:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: l_comment ASC
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=1 row-size=263B cardinality=6001215
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+01:SORT
+| order by: l_comment ASC
+| mem-estimate=160.00MB mem-reservation=48.00MB
+| tuple-ids=1 row-size=263B cardinality=6001215
+|
+00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
+ partitions=1/1 files=3 size=193.61MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=80.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=96.00MB
+Per-Host Resource Estimates: Memory=480.00MB
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+02:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: l_comment ASC
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=1 row-size=263B cardinality=6001215
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+01:SORT
+| order by: l_comment ASC
+| mem-estimate=160.00MB mem-reservation=48.00MB
+| tuple-ids=1 row-size=263B cardinality=6001215
+|
+00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
+ partitions=1/1 files=3 size=193.61MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=80.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+====
+# TOP-N
+select *
+from tpch_parquet.lineitem
+order by l_comment
+limit 100
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=80.03MB
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+02:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: l_comment ASC
+| limit: 100
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=1 row-size=263B cardinality=100
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+01:TOP-N [LIMIT=100]
+| order by: l_comment ASC
+| mem-estimate=25.66KB mem-reservation=0B
+| tuple-ids=1 row-size=263B cardinality=100
+|
+00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
+ partitions=1/1 files=3 size=193.61MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=80.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=160.05MB
+
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+02:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: l_comment ASC
+| limit: 100
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=1 row-size=263B cardinality=100
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+01:TOP-N [LIMIT=100]
+| order by: l_comment ASC
+| mem-estimate=25.66KB mem-reservation=0B
+| tuple-ids=1 row-size=263B cardinality=100
+|
+00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
+ partitions=1/1 files=3 size=193.61MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=80.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+====
+# Hash Join
+select *
+from tpch.lineitem inner join tpch.orders on l_orderkey = o_orderkey
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=136.00MB
+Per-Host Resource Estimates: Memory=388.41MB
+
+F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+04:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0,1 row-size=454B cardinality=5757710
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+02:HASH JOIN [INNER JOIN, BROADCAST]
+| hash predicates: l_orderkey = o_orderkey
+| runtime filters: RF000 <- o_orderkey
+| mem-estimate=300.41MB mem-reservation=136.00MB
+| tuple-ids=0,1 row-size=454B cardinality=5757710
+|
+|--03:EXCHANGE [BROADCAST]
+| | mem-estimate=0B mem-reservation=0B
+| | tuple-ids=1 row-size=191B cardinality=1500000
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
+| 01:SCAN HDFS [tpch.orders, RANDOM]
+| partitions=1/1 files=1 size=162.56MB
+| table stats: 1500000 rows total
+| column stats: all
+| mem-estimate=88.00MB mem-reservation=0B
+| tuple-ids=1 row-size=191B cardinality=1500000
+|
+00:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
+ runtime filters: RF000 -> l_orderkey
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=272.00MB
+Per-Host Resource Estimates: Memory=952.83MB
+
+F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+04:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0,1 row-size=454B cardinality=5757710
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+02:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=00
+| hash predicates: l_orderkey = o_orderkey
+| runtime filters: RF000 <- o_orderkey
+| mem-estimate=300.41MB mem-reservation=136.00MB
+| tuple-ids=0,1 row-size=454B cardinality=5757710
+|
+|--F03:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: o_orderkey
+| | mem-estimate=0B mem-reservation=0B
+| |
+| 03:EXCHANGE [BROADCAST]
+| | mem-estimate=0B mem-reservation=0B
+| | tuple-ids=1 row-size=191B cardinality=1500000
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
+| 01:SCAN HDFS [tpch.orders, RANDOM]
+| partitions=1/1 files=1 size=162.56MB
+| table stats: 1500000 rows total
+| column stats: all
+| mem-estimate=88.00MB mem-reservation=0B
+| tuple-ids=1 row-size=191B cardinality=1500000
+|
+00:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
+ runtime filters: RF000 -> l_orderkey
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+====
+# Nested loop join
+select *
+from tpch.lineitem, tpch.orders
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=361.10MB
+
+F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+04:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0,1 row-size=454B cardinality=9001822500000
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
+| mem-estimate=273.10MB mem-reservation=0B
+| tuple-ids=0,1 row-size=454B cardinality=9001822500000
+|
+|--03:EXCHANGE [BROADCAST]
+| | mem-estimate=0B mem-reservation=0B
+| | tuple-ids=1 row-size=191B cardinality=1500000
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
+| 01:SCAN HDFS [tpch.orders, RANDOM]
+| partitions=1/1 files=1 size=162.56MB
+| table stats: 1500000 rows total
+| column stats: all
+| mem-estimate=88.00MB mem-reservation=0B
+| tuple-ids=1 row-size=191B cardinality=1500000
+|
+00:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=898.21MB
+
+F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+04:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0,1 row-size=454B cardinality=9001822500000
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
+| join table id: 00
+| mem-estimate=273.10MB mem-reservation=0B
+| tuple-ids=0,1 row-size=454B cardinality=9001822500000
+|
+|--F03:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | mem-estimate=0B mem-reservation=0B
+| |
+| 03:EXCHANGE [BROADCAST]
+| | mem-estimate=0B mem-reservation=0B
+| | tuple-ids=1 row-size=191B cardinality=1500000
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
+| 01:SCAN HDFS [tpch.orders, RANDOM]
+| partitions=1/1 files=1 size=162.56MB
+| table stats: 1500000 rows total
+| column stats: all
+| mem-estimate=88.00MB mem-reservation=0B
+| tuple-ids=1 row-size=191B cardinality=1500000
+|
+00:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
+====
+# Empty set node
+select * from functional.alltypes where 1 = 2
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=10.00MB
+
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+00:EMPTYSET
+ mem-estimate=0B mem-reservation=0B
+ tuple-ids=0 row-size=0B cardinality=0
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=10.00MB
+
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+00:EMPTYSET
+ mem-estimate=0B mem-reservation=0B
+ tuple-ids=0 row-size=0B cardinality=0
+====
+# Analytic function
+select max(tinyint_col) over(partition by int_col)
+from functional.alltypes
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=40.00MB
+Per-Host Resource Estimates: Memory=24.00MB
+
+F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+04:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=3,2 row-size=6B cardinality=7300
+|
+F01:PLAN FRAGMENT [HASH(int_col)] hosts=3 instances=3
+02:ANALYTIC
+| functions: max(tinyint_col)
+| partition by: int_col
+| mem-estimate=0B mem-reservation=16.00MB
+| tuple-ids=3,2 row-size=6B cardinality=7300
+|
+01:SORT
+| order by: int_col ASC NULLS FIRST
+| mem-estimate=8.00MB mem-reservation=24.00MB
+| tuple-ids=3 row-size=5B cardinality=7300
+|
+03:EXCHANGE [HASH(int_col)]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0 row-size=5B cardinality=7300
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+00:SCAN HDFS [functional.alltypes, RANDOM]
+ partitions=24/24 files=24 size=478.45KB
+ table stats: 7300 rows total
+ column stats: all
+ mem-estimate=16.00MB mem-reservation=0B
+ tuple-ids=0 row-size=5B cardinality=7300
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=80.00MB
+Per-Host Resource Estimates: Memory=48.00MB
+
+F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+04:EXCHANGE [UNPARTITIONED]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=3,2 row-size=6B cardinality=7300
+|
+F01:PLAN FRAGMENT [HASH(int_col)] hosts=3 instances=6
+02:ANALYTIC
+| functions: max(tinyint_col)
+| partition by: int_col
+| mem-estimate=0B mem-reservation=16.00MB
+| tuple-ids=3,2 row-size=6B cardinality=7300
+|
+01:SORT
+| order by: int_col ASC NULLS FIRST
+| mem-estimate=8.00MB mem-reservation=24.00MB
+| tuple-ids=3 row-size=5B cardinality=7300
+|
+03:EXCHANGE [HASH(int_col)]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0 row-size=5B cardinality=7300
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+00:SCAN HDFS [functional.alltypes, RANDOM]
+ partitions=24/24 files=24 size=478.45KB
+ table stats: 7300 rows total
+ column stats: all
+ mem-estimate=16.00MB mem-reservation=0B
+ tuple-ids=0 row-size=5B cardinality=7300
+====
+# TPC-H Q18: Pipeline of joins with some non-trivial subtrees on right side of joins
+ select
+ c_name,
+ c_custkey,
+ o_orderkey,
+ o_orderdate,
+ o_totalprice,
+ sum(l_quantity)
+from
+ tpch.customer,
+ tpch.orders,
+ tpch.lineitem
+where
+ o_orderkey in (
+ select
+ l_orderkey
+ from
+ tpch.lineitem
+ group by
+ l_orderkey
+ having
+ sum(l_quantity) > 300
+ )
+ and c_custkey = o_custkey
+ and o_orderkey = l_orderkey
+group by
+ c_name,
+ c_custkey,
+ o_orderkey,
+ o_orderdate,
+ o_totalprice
+order by
+ o_totalprice desc,
+ o_orderdate
+limit 100
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=672.00MB
+Per-Host Resource Estimates: Memory=242.95MB
+
+F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+17:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: o_totalprice DESC, o_orderdate ASC
+| limit: 100
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=7 row-size=100B cardinality=100
+|
+F06:PLAN FRAGMENT [HASH(c_name,c_custkey,o_orderkey,o_orderdate,o_totalprice)] hosts=3 instances=3
+09:TOP-N [LIMIT=100]
+| order by: o_totalprice DESC, o_orderdate ASC
+| mem-estimate=9.77KB mem-reservation=0B
+| tuple-ids=7 row-size=100B cardinality=100
+|
+16:AGGREGATE [FINALIZE]
+| output: sum:merge(l_quantity)
+| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
+| mem-estimate=60.40MB mem-reservation=264.00MB
+| tuple-ids=6 row-size=100B cardinality=575772
+|
+15:EXCHANGE [HASH(c_name,c_custkey,o_orderkey,o_orderdate,o_totalprice)]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=6 row-size=100B cardinality=575772
+|
+F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
+08:AGGREGATE [STREAMING]
+| output: sum(l_quantity)
+| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
+| mem-estimate=60.40MB mem-reservation=0B
+| tuple-ids=6 row-size=100B cardinality=575772
+|
+07:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]
+| hash predicates: o_orderkey = l_orderkey
+| runtime filters: RF000 <- l_orderkey
+| mem-estimate=1.31MB mem-reservation=136.00MB
+| tuple-ids=2,1,0 row-size=108B cardinality=575772
+|
+|--14:AGGREGATE [FINALIZE]
+| | output: sum:merge(l_quantity)
+| | group by: l_orderkey
+| | having: sum(l_quantity) > 300
+| | mem-estimate=10.00MB mem-reservation=264.00MB
+| | tuple-ids=4 row-size=24B cardinality=156344
+| |
+| 13:EXCHANGE [HASH(l_orderkey)]
+| | mem-estimate=0B mem-reservation=0B
+| | tuple-ids=4 row-size=24B cardinality=1563438
+| |
+| F04:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+| 04:AGGREGATE [STREAMING]
+| | output: sum(l_quantity)
+| | group by: l_orderkey
+| | mem-estimate=39.36MB mem-reservation=0B
+| | tuple-ids=4 row-size=24B cardinality=1563438
+| |
+| 03:SCAN HDFS [tpch.lineitem, RANDOM]
+| partitions=1/1 files=1 size=718.94MB
+| table stats: 6001215 rows total
+| column stats: all
+| mem-estimate=88.00MB mem-reservation=0B
+| tuple-ids=3 row-size=16B cardinality=6001215
+|
+06:HASH JOIN [INNER JOIN, BROADCAST]
+| hash predicates: o_custkey = c_custkey
+| runtime filters: RF001 <- c_custkey
+| mem-estimate=6.61MB mem-reservation=136.00MB
+| tuple-ids=2,1,0 row-size=108B cardinality=5757710
+|
+|--12:EXCHANGE [BROADCAST]
+| | mem-estimate=0B mem-reservation=0B
+| | tuple-ids=0 row-size=42B cardinality=150000
+| |
+| F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| 00:SCAN HDFS [tpch.customer, RANDOM]
+| partitions=1/1 files=1 size=23.08MB
+| table stats: 150000 rows total
+| column stats: all
+| mem-estimate=32.00MB mem-reservation=0B
+| tuple-ids=0 row-size=42B cardinality=150000
+|
+05:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash predicates: l_orderkey = o_orderkey
+| runtime filters: RF002 <- o_orderkey
+| mem-estimate=26.23MB mem-reservation=136.00MB
+| tuple-ids=2,1 row-size=66B cardinality=5757710
+|
+|--11:EXCHANGE [HASH(o_orderkey)]
+| | mem-estimate=0B mem-reservation=0B
+| | tuple-ids=1 row-size=50B cardinality=1500000
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
+| 01:SCAN HDFS [tpch.orders, RANDOM]
+| partitions=1/1 files=1 size=162.56MB
+| runtime filters: RF000 -> o_orderkey, RF001 -> o_custkey
+| table stats: 1500000 rows total
+| column stats: all
+| mem-estimate=88.00MB mem-reservation=0B
+| tuple-ids=1 row-size=50B cardinality=1500000
+|
+10:EXCHANGE [HASH(l_orderkey)]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=2 row-size=16B cardinality=6001215
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+02:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
+ runtime filters: RF000 -> tpch.lineitem.l_orderkey, RF002 -> l_orderkey
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=2 row-size=16B cardinality=6001215
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=1.83GB
+Per-Host Resource Estimates: Memory=973.08MB
+
+F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B
+|
+17:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: o_totalprice DESC, o_orderdate ASC
+| limit: 100
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=7 row-size=100B cardinality=100
+|
+F06:PLAN FRAGMENT [HASH(c_name,c_custkey,o_orderkey,o_orderdate,o_totalprice)] hosts=3 instances=6
+09:TOP-N [LIMIT=100]
+| order by: o_totalprice DESC, o_orderdate ASC
+| mem-estimate=9.77KB mem-reservation=0B
+| tuple-ids=7 row-size=100B cardinality=100
+|
+16:AGGREGATE [FINALIZE]
+| output: sum:merge(l_quantity)
+| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
+| mem-estimate=60.40MB mem-reservation=264.00MB
+| tuple-ids=6 row-size=100B cardinality=575772
+|
+15:EXCHANGE [HASH(c_name,c_custkey,o_orderkey,o_orderdate,o_totalprice)]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=6 row-size=100B cardinality=575772
+|
+F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
+08:AGGREGATE [STREAMING]
+| output: sum(l_quantity)
+| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
+| mem-estimate=60.40MB mem-reservation=0B
+| tuple-ids=6 row-size=100B cardinality=575772
+|
+07:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]
+| hash-table-id=00
+| hash predicates: o_orderkey = l_orderkey
+| runtime filters: RF000 <- l_orderkey
+| mem-estimate=671.79KB mem-reservation=136.00MB
+| tuple-ids=2,1,0 row-size=108B cardinality=575772
+|
+|--F08:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: l_orderkey
+| | mem-estimate=0B mem-reservation=0B
+| |
+| 14:AGGREGATE [FINALIZE]
+| | output: sum:merge(l_quantity)
+| | group by: l_orderkey
+| | having: sum(l_quantity) > 300
+| | mem-estimate=10.00MB mem-reservation=264.00MB
+| | tuple-ids=4 row-size=24B cardinality=156344
+| |
+| 13:EXCHANGE [HASH(l_orderkey)]
+| | mem-estimate=0B mem-reservation=0B
+| | tuple-ids=4 row-size=24B cardinality=1563438
+| |
+| F04:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+| 04:AGGREGATE [STREAMING]
+| | output: sum(l_quantity)
+| | group by: l_orderkey
+| | mem-estimate=39.36MB mem-reservation=0B
+| | tuple-ids=4 row-size=24B cardinality=1563438
+| |
+| 03:SCAN HDFS [tpch.lineitem, RANDOM]
+| partitions=1/1 files=1 size=718.94MB
+| table stats: 6001215 rows total
+| column stats: all
+| mem-estimate=88.00MB mem-reservation=0B
+| tuple-ids=3 row-size=16B cardinality=6001215
+|
+06:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=01
+| hash predicates: o_custkey = c_custkey
+| runtime filters: RF001 <- c_custkey
+| mem-estimate=6.61MB mem-reservation=136.00MB
+| tuple-ids=2,1,0 row-size=108B cardinality=5757710
+|
+|--F09:PLAN FRAGMENT [HASH(l_orderkey)] hosts=1 instances=2
+| JOIN BUILD
+| | join-table-id=01 plan-id=02 cohort-id=01
+| | build expressions: c_custkey
+| | mem-estimate=0B mem-reservation=0B
+| |
+| 12:EXCHANGE [BROADCAST]
+| | mem-estimate=0B mem-reservation=0B
+| | tuple-ids=0 row-size=42B cardinality=150000
+| |
+| F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
+| 00:SCAN HDFS [tpch.customer, RANDOM]
+| partitions=1/1 files=1 size=23.08MB
+| table stats: 150000 rows total
+| column stats: all
+| mem-estimate=32.00MB mem-reservation=0B
+| tuple-ids=0 row-size=42B cardinality=150000
+|
+05:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=02
+| hash predicates: l_orderkey = o_orderkey
+| runtime filters: RF002 <- o_orderkey
+| mem-estimate=13.11MB mem-reservation=136.00MB
+| tuple-ids=2,1 row-size=66B cardinality=5757710
+|
+|--F10:PLAN FRAGMENT [HASH(l_orderkey)] hosts=2 instances=4
+| JOIN BUILD
+| | join-table-id=02 plan-id=03 cohort-id=01
+| | build expressions: o_orderkey
+| | mem-estimate=0B mem-reservation=0B
+| |
+| 11:EXCHANGE [HASH(o_orderkey)]
+| | mem-estimate=0B mem-reservation=0B
+| | tuple-ids=1 row-size=50B cardinality=1500000
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
+| 01:SCAN HDFS [tpch.orders, RANDOM]
+| partitions=1/1 files=1 size=162.56MB
+| runtime filters: RF000 -> o_orderkey, RF001 -> o_custkey
+| table stats: 1500000 rows total
+| column stats: all
+| mem-estimate=88.00MB mem-reservation=0B
+| tuple-ids=1 row-size=50B cardinality=1500000
+|
+10:EXCHANGE [HASH(l_orderkey)]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=2 row-size=16B cardinality=6001215
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+02:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
+ runtime filters: RF000 -> tpch.lineitem.l_orderkey, RF002 -> l_orderkey
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=2 row-size=16B cardinality=6001215
+====
+# Unpartitioned HDFS insert
+insert into table functional.alltypesnopart
+select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col,
+float_col, double_col, date_string_col, string_col, timestamp_col
+from functional.alltypes
+where year=2009 and month=05
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=16.03MB
+
+F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+WRITE TO HDFS [functional.alltypesnopart, OVERWRITE=false]
+| partitions=1
+| mem-estimate=26.94KB mem-reservation=0B
+|
+00:SCAN HDFS [functional.alltypes, RANDOM]
+ partitions=1/24 files=1 size=20.36KB
+ table stats: 7300 rows total
+ column stats: all
+ mem-estimate=16.00MB mem-reservation=0B
+ tuple-ids=0 row-size=89B cardinality=310
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=32.03MB
+
+F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
+WRITE TO HDFS [functional.alltypesnopart, OVERWRITE=false]
+| partitions=1
+| mem-estimate=13.47KB mem-reservation=0B
+|
+00:SCAN HDFS [functional.alltypes, RANDOM]
+ partitions=1/24 files=1 size=20.36KB
+ table stats: 7300 rows total
+ column stats: all
+ mem-estimate=16.00MB mem-reservation=0B
+ tuple-ids=0 row-size=89B cardinality=310
+====
+# Dynamically partitioned HDFS insert
+create table dummy_insert
+partitioned by (l_partkey) as
+select l_comment, l_partkey from tpch.lineitem
+---- DISTRIBUTEDPLAN
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=184.33MB
+
+F01:PLAN FRAGMENT [HASH(l_partkey)] hosts=3 instances=3
+WRITE TO HDFS [default.dummy_insert, OVERWRITE=false, PARTITION-KEYS=(l_partkey)]
+| partitions=200516
+| mem-estimate=96.33MB mem-reservation=0B
+|
+01:EXCHANGE [HASH(l_partkey)]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0 row-size=50B cardinality=6001215
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+00:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=50B cardinality=6001215
+---- PARALLELPLANS
+Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=272.33MB
+
+F01:PLAN FRAGMENT [HASH(l_partkey)] hosts=3 instances=6
+WRITE TO HDFS [default.dummy_insert, OVERWRITE=false, PARTITION-KEYS=(l_partkey)]
+| partitions=200516
+| mem-estimate=48.16MB mem-reservation=0B
+|
+01:EXCHANGE [HASH(l_partkey)]
+| mem-estimate=0B mem-reservation=0B
+| tuple-ids=0 row-size=50B cardinality=6001215
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
+00:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
+ table stats: 6001215 rows total
+ column stats: all
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=50B cardinality=6001215
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
index 0cb0610..06d76c8 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
@@ -1,117 +1,19 @@
====
---- QUERY
-# Tests explaining a query (TPCDS-Q19)
+# Explain a simple hash join query.
explain
-select
- i_brand_id,
- i_brand,
- i_manufact_id,
- i_manufact,
- sum(ss_ext_sales_price) ext_price
-from
- tpcds.store_sales
- join tpcds.item on (store_sales.ss_item_sk = item.i_item_sk)
- join tpcds.customer on (store_sales.ss_customer_sk = customer.c_customer_sk)
- join tpcds.customer_address on (customer.c_current_addr_sk = customer_address.ca_address_sk)
- join tpcds.store on (store_sales.ss_store_sk = store.s_store_sk)
-where
- ss_date between '1999-11-01' and '1999-11-30'
- and i_manager_id = 7
- and substr(ca_zip, 1, 5) <> substr(s_zip, 1, 5)
-group by
- i_brand,
- i_brand_id,
- i_manufact_id,
- i_manufact
-order by
- ext_price desc,
- i_brand,
- i_brand_id,
- i_manufact_id,
- i_manufact
-limit 100
----- RESULTS
-'Estimated Per-Host Requirements: Memory=72.66MB VCores=5'
+select *
+from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
+---- RESULTS: VERIFY_IS_EQUAL
+'Per-Host Resource Reservation: Memory=136.00MB'
+'Per-Host Resource Estimates: Memory=388.41MB'
''
-'19:TOP-N [LIMIT=100]'
-'18:EXCHANGE [UNPARTITIONED]'
-'10:TOP-N [LIMIT=100]'
-'17:AGGREGATE [MERGE FINALIZE]'
-'16:EXCHANGE [HASH(i_brand,i_brand_id,i_manufact_id,i_manufact)]'
-'09:AGGREGATE'
-'08:HASH JOIN [INNER JOIN, BROADCAST]'
-'|--15:EXCHANGE [BROADCAST]'
-'| 04:SCAN HDFS [tpcds.store]'
-'07:HASH JOIN [INNER JOIN, BROADCAST]'
-'|--14:EXCHANGE [BROADCAST]'
-'| 01:SCAN HDFS [tpcds.item]'
-'06:HASH JOIN [INNER JOIN, BROADCAST]'
-'|--13:EXCHANGE [BROADCAST]'
-'| 00:SCAN HDFS [tpcds.store_sales]'
-'05:HASH JOIN [INNER JOIN, PARTITIONED]'
-'|--12:EXCHANGE [HASH(customer.c_current_addr_sk)]'
-'| 02:SCAN HDFS [tpcds.customer]'
-'11:EXCHANGE [HASH(customer_address.ca_address_sk)]'
-'03:SCAN HDFS [tpcds.customer_address]'
-====
----- QUERY
-# Tests explaining an insert query
-explain insert overwrite functional.alltypessmall (id, string_col)
-partition (year, month)
-select a.id, a.string_col, a.year, a.month from functional.alltypes a
- left semi join functional.alltypesagg b on (a.id = b.id)
-where a.year > 2009 and a.month = 4
-union distinct
-select id, string_col, year, month from functional.alltypes
----- RESULTS
-'Estimated Per-Host Requirements: Memory=30.00MB VCores=3'
-''
-'WRITE TO HDFS [functional.alltypessmall, OVERWRITE=true, PARTITION-KEYS=(year,month)]'
-'14:EXCHANGE [HASH(year,month)]'
-'13:AGGREGATE [MERGE FINALIZE]'
-'12:EXCHANGE [HASH(id,string_col,year,month)]'
-'|--11:AGGREGATE'
-'| 10:MERGE'
-'| 04:SCAN HDFS [functional.alltypes]'
-'05:AGGREGATE'
-'09:MERGE'
-'03:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]'
-'|--07:EXCHANGE [HASH(b.id)]'
-'| 02:SCAN HDFS [functional.alltypesagg b]'
-'06:EXCHANGE [HASH(a.id)]'
-'01:SCAN HDFS [functional.alltypes a]'
-====
----- QUERY
-# Tests explaining an insert query to/from an HBase table
-explain insert into functional_hbase.alltypes
-select a.* from functional_hbase.alltypessmall a
- cross join functional.alltypessmall b
-where a.year > 2009 and a.month = 4
-union all
-select * from functional_hbase.alltypessmall
----- RESULTS
-'Estimated Per-Host Requirements: Memory=1.02GB VCores=3'
-''
-'WRITE TO HBASE table=functional_hbase.alltypes'
-'06:EXCHANGE [UNPARTITIONED]'
-'|--08:MERGE'
-'| 04:SCAN HBASE [functional_hbase.alltypessmall]'
-'07:MERGE'
-'03:CROSS JOIN [BROADCAST]'
-'|--05:EXCHANGE [BROADCAST]'
-'| 02:SCAN HDFS [functional.alltypessmall b]'
-'01:SCAN HBASE [functional_hbase.alltypessmall a]'
-====
----- QUERY
-# Tests explaining an CTAS statement.
-explain create table t as
-select * from functional.alltypes
-where month = 2
----- RESULTS
-'Estimated Per-Host Requirements: Memory=16.02MB VCores=1'
-''
-'WRITE TO HDFS [functional.t, OVERWRITE=false]'
-'00:SCAN HDFS [functional.alltypes]'
+'PLAN-ROOT SINK'
+'04:EXCHANGE [UNPARTITIONED]'
+'02:HASH JOIN [INNER JOIN, BROADCAST]'
+'|--03:EXCHANGE [BROADCAST]'
+'| 01:SCAN HDFS [tpch.orders]'
+'00:SCAN HDFS [tpch.lineitem]'
====
---- QUERY
# Tests the warning about missing table stats in the explain header.
@@ -120,19 +22,7 @@ from functional_avro.alltypes t1
inner join functional_parquet.alltypessmall t2 on (t1.id = t2.id)
left outer join functional_avro.alltypes t3 on (t2.id = t3.id)
where t1.month = 1 and t2.year = 2009 and t3.bool_col = false
----- RESULTS
-'Estimated Per-Host Requirements: Memory=4.03GB VCores=3'
+---- RESULTS: VERIFY_IS_SUBSET
'WARNING: The following tables are missing relevant table and/or column statistics.'
'functional_avro.alltypes, functional_parquet.alltypessmall'
-''
-'09:AGGREGATE [MERGE FINALIZE]'
-'08:EXCHANGE [UNPARTITIONED]'
-'05:AGGREGATE'
-'04:HASH JOIN [LEFT OUTER JOIN, BROADCAST]'
-'|--07:EXCHANGE [BROADCAST]'
-'| 02:SCAN HDFS [functional_avro.alltypes t3]'
-'03:HASH JOIN [INNER JOIN, BROADCAST]'
-'|--06:EXCHANGE [BROADCAST]'
-'| 01:SCAN HDFS [functional_parquet.alltypessmall t2]'
-'00:SCAN HDFS [functional_avro.alltypes t1]'
====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
index a1a172d..a082603 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
@@ -1,187 +1,29 @@
====
---- QUERY
-# Tests explaining a query (TPCDS-Q19)
+# Explain a simple hash join query.
explain
-select
- i_brand_id,
- i_brand,
- i_manufact_id,
- i_manufact,
- sum(ss_ext_sales_price) ext_price
-from
- tpcds.store_sales
- join tpcds.item on (store_sales.ss_item_sk = item.i_item_sk)
- join tpcds.customer on (store_sales.ss_customer_sk = customer.c_customer_sk)
- join tpcds.customer_address on (customer.c_current_addr_sk = customer_address.ca_address_sk)
- join tpcds.store on (store_sales.ss_store_sk = store.s_store_sk)
-where
- ss_date between '1999-11-01' and '1999-11-30'
- and i_manager_id = 7
- and substr(ca_zip, 1, 5) <> substr(s_zip, 1, 5)
-group by
- i_brand,
- i_brand_id,
- i_manufact_id,
- i_manufact
-order by
- ext_price desc,
- i_brand,
- i_brand_id,
- i_manufact_id,
- i_manufact
-limit 100
----- RESULTS
-'Estimated Per-Host Requirements: Memory=72.66MB VCores=5'
+select *
+from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
+---- RESULTS: VERIFY_IS_EQUAL
+'Per-Host Resource Reservation: Memory=136.00MB'
+'Per-Host Resource Estimates: Memory=388.41MB'
''
-'19:TOP-N [LIMIT=100]'
-'| order by: sum(ss_ext_sales_price) DESC, i_brand ASC, i_brand_id ASC, i_manufact_id ASC, i_manufact ASC'
+'PLAN-ROOT SINK'
'|'
-'18:EXCHANGE [UNPARTITIONED]'
+'04:EXCHANGE [UNPARTITIONED]'
'|'
-'10:TOP-N [LIMIT=100]'
-'| order by: sum(ss_ext_sales_price) DESC, i_brand ASC, i_brand_id ASC, i_manufact_id ASC, i_manufact ASC'
+'02:HASH JOIN [INNER JOIN, BROADCAST]'
+'| hash predicates: l_orderkey = o_orderkey'
+'| runtime filters: RF000 <- o_orderkey'
'|'
-'17:AGGREGATE [MERGE FINALIZE]'
-'| output: sum(sum(ss_ext_sales_price))'
-'| group by: i_brand, i_brand_id, i_manufact_id, i_manufact'
-'|'
-'16:EXCHANGE [HASH(i_brand,i_brand_id,i_manufact_id,i_manufact)]'
-'|'
-'09:AGGREGATE'
-'| output: sum(ss_ext_sales_price)'
-'| group by: i_brand, i_brand_id, i_manufact_id, i_manufact'
-'|'
-'08:HASH JOIN [INNER JOIN, BROADCAST]'
-'| hash predicates: store_sales.ss_store_sk = store.s_store_sk'
-'| other predicates: substr(ca_zip, 1, 5) != substr(s_zip, 1, 5)'
-'|'
-'|--15:EXCHANGE [BROADCAST]'
-'| |'
-'| 04:SCAN HDFS [tpcds.store]'
-'| partitions=1/1 size=3.08KB'
-'|'
-'07:HASH JOIN [INNER JOIN, BROADCAST]'
-'| hash predicates: store_sales.ss_item_sk = item.i_item_sk'
-'|'
-'|--14:EXCHANGE [BROADCAST]'
+'|--03:EXCHANGE [BROADCAST]'
'| |'
-'| 01:SCAN HDFS [tpcds.item]'
-'| partitions=1/1 size=4.82MB'
-'| predicates: i_manager_id = 7'
-'|'
-'06:HASH JOIN [INNER JOIN, BROADCAST]'
-'| hash predicates: customer.c_customer_sk = store_sales.ss_customer_sk'
-'|'
-'|--13:EXCHANGE [BROADCAST]'
-'| |'
-'| 00:SCAN HDFS [tpcds.store_sales]'
-'| partitions=2/120 size=663.52KB'
-'|'
-'05:HASH JOIN [INNER JOIN, PARTITIONED]'
-'| hash predicates: customer_address.ca_address_sk = customer.c_current_addr_sk'
-'|'
-'|--12:EXCHANGE [HASH(customer.c_current_addr_sk)]'
-'| |'
-'| 02:SCAN HDFS [tpcds.customer]'
-'| partitions=1/1 size=12.60MB'
-'|'
-'11:EXCHANGE [HASH(customer_address.ca_address_sk)]'
+'| 01:SCAN HDFS [tpch.orders]'
+row_regex:.*partitions=1/1 files=1 size=.*
'|'
-'03:SCAN HDFS [tpcds.customer_address]'
-' partitions=1/1 size=5.25MB'
-====
----- QUERY
-# Tests explaining an insert query
-explain insert overwrite functional.alltypessmall (id, string_col)
-partition (year, month)
-select a.id, a.string_col, a.year, a.month from functional.alltypes a
- left semi join functional.alltypesagg b on (a.id = b.id)
-where a.year > 2009 and a.month = 4
-union distinct
-select id, string_col, year, month from functional.alltypes
----- RESULTS
-'Estimated Per-Host Requirements: Memory=30.00MB VCores=3'
-''
-'WRITE TO HDFS [functional.alltypessmall, OVERWRITE=true, PARTITION-KEYS=(year,month)]'
-'| partitions=96'
-'|'
-'14:EXCHANGE [HASH(year,month)]'
-'|'
-'13:AGGREGATE [MERGE FINALIZE]'
-'| group by: id, string_col, year, month'
-'|'
-'12:EXCHANGE [HASH(id,string_col,year,month)]'
-'|'
-'|--11:AGGREGATE'
-'| | group by: id, string_col, year, month'
-'| |'
-'| 10:MERGE'
-'| |'
-'| 04:SCAN HDFS [functional.alltypes]'
-'| partitions=24/24 size=478.45KB'
-'|'
-'05:AGGREGATE'
-'| group by: id, string_col, year, month'
-'|'
-'09:MERGE'
-'|'
-'03:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]'
-'| hash predicates: a.id = b.id'
-'|'
-'|--07:EXCHANGE [HASH(b.id)]'
-'| |'
-'| 02:SCAN HDFS [functional.alltypesagg b]'
-'| partitions=10/10 size=743.67KB'
-'|'
-'06:EXCHANGE [HASH(a.id)]'
-'|'
-'01:SCAN HDFS [functional.alltypes a]'
-' partitions=1/24 size=19.71KB'
-====
----- QUERY
-# Tests explaining an insert query to/from an HBase table
-explain insert into functional_hbase.alltypes
-select a.* from functional_hbase.alltypessmall a
- cross join functional.alltypessmall b
-where a.year > 2009 and a.month = 4
-union all
-select * from functional_hbase.alltypessmall
----- RESULTS
-'Estimated Per-Host Requirements: Memory=1.02GB VCores=3'
-''
-'WRITE TO HBASE table=functional_hbase.alltypes'
-'|'
-'06:EXCHANGE [UNPARTITIONED]'
-'|'
-'|--08:MERGE'
-'| |'
-'| 04:SCAN HBASE [functional_hbase.alltypessmall]'
-'|'
-'07:MERGE'
-'|'
-'03:CROSS JOIN [BROADCAST]'
-'|'
-'|--05:EXCHANGE [BROADCAST]'
-'| |'
-'| 02:SCAN HDFS [functional.alltypessmall b]'
-'| partitions=4/4 size=6.32KB'
-'|'
-'01:SCAN HBASE [functional_hbase.alltypessmall a]'
-' predicates: a.year > 2009, a.month = 4'
-====
----- QUERY
-# Tests explaining an CTAS statement.
-explain create table t as
-select * from functional.alltypes
-where month = 2
----- RESULTS
-'Estimated Per-Host Requirements: Memory=16.02MB VCores=1'
-''
-'WRITE TO HDFS [functional.t, OVERWRITE=false]'
-'| partitions=1'
-'|'
-'00:SCAN HDFS [functional.alltypes]'
-' partitions=2/24 size=36.51KB'
+'00:SCAN HDFS [tpch.lineitem]'
+row_regex:.*partitions=1/1 files=1 size=.*
+' runtime filters: RF000 -> l_orderkey'
====
---- QUERY
# Tests the warning about missing table stats in the explain header.
@@ -190,37 +32,7 @@ from functional_avro.alltypes t1
inner join functional_parquet.alltypessmall t2 on (t1.id = t2.id)
left outer join functional_avro.alltypes t3 on (t2.id = t3.id)
where t1.month = 1 and t2.year = 2009 and t3.bool_col = false
----- RESULTS
-'Estimated Per-Host Requirements: Memory=4.03GB VCores=3'
+---- RESULTS: VERIFY_IS_SUBSET
'WARNING: The following tables are missing relevant table and/or column statistics.'
'functional_avro.alltypes, functional_parquet.alltypessmall'
-''
-'09:AGGREGATE [MERGE FINALIZE]'
-'| output: sum(count(t1.int_col)), sum(sum(t2.float_col)), sum(count(t2.float_col)), sum(sum(t3.bigint_col))'
-'|'
-'08:EXCHANGE [UNPARTITIONED]'
-'|'
-'05:AGGREGATE'
-'| output: count(t1.int_col), sum(t2.float_col), count(t2.float_col), sum(t3.bigint_col)'
-'|'
-'04:HASH JOIN [LEFT OUTER JOIN, BROADCAST]'
-'| hash predicates: t2.id = t3.id'
-'| other predicates: t3.bool_col = FALSE'
-'|'
-'|--07:EXCHANGE [BROADCAST]'
-'| |'
-'| 02:SCAN HDFS [functional_avro.alltypes t3]'
-'| partitions=24/24 size=470.35KB'
-'| predicates: t3.bool_col = FALSE'
-'|'
-'03:HASH JOIN [INNER JOIN, BROADCAST]'
-'| hash predicates: t1.id = t2.id'
-'|'
-'|--06:EXCHANGE [BROADCAST]'
-'| |'
-'| 01:SCAN HDFS [functional_parquet.alltypessmall t2]'
-'| partitions=4/4 size=9.63KB'
-'|'
-'00:SCAN HDFS [functional_avro.alltypes t1]'
-' partitions=2/24 size=39.87KB'
====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
index 2206ae8..32171fb 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
@@ -1,298 +1,47 @@
====
---- QUERY
-# Tests explaining a query (TPCDS-Q19)
+# Explain a simple hash join query.
explain
-select
- i_brand_id,
- i_brand,
- i_manufact_id,
- i_manufact,
- sum(ss_ext_sales_price) ext_price
-from
- tpcds.store_sales
- join tpcds.item on (store_sales.ss_item_sk = item.i_item_sk)
- join tpcds.customer on (store_sales.ss_customer_sk = customer.c_customer_sk)
- join tpcds.customer_address on (customer.c_current_addr_sk = customer_address.ca_address_sk)
- join tpcds.store on (store_sales.ss_store_sk = store.s_store_sk)
-where
- ss_date between '1999-11-01' and '1999-11-30'
- and i_manager_id = 7
- and substr(ca_zip, 1, 5) <> substr(s_zip, 1, 5)
-group by
- i_brand,
- i_brand_id,
- i_manufact_id,
- i_manufact
-order by
- ext_price desc,
- i_brand,
- i_brand_id,
- i_manufact_id,
- i_manufact
-limit 100
----- RESULTS
-'Estimated Per-Host Requirements: Memory=72.66MB VCores=5'
+select *
+from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
+---- RESULTS: VERIFY_IS_EQUAL
+'Per-Host Resource Reservation: Memory=136.00MB'
+'Per-Host Resource Estimates: Memory=388.41MB'
''
-'19:TOP-N [LIMIT=100]'
-'| order by: sum(ss_ext_sales_price) DESC, i_brand ASC, i_brand_id ASC, i_manufact_id ASC, i_manufact ASC'
-'| hosts=3 per-host-mem=unavailable'
-'| tuple-ids=5 row-size=48B cardinality=100'
-'|'
-'18:EXCHANGE [UNPARTITIONED]'
-'| hosts=3 per-host-mem=unavailable'
-'| tuple-ids=5 row-size=48B cardinality=100'
-'|'
-'10:TOP-N [LIMIT=100]'
-'| order by: sum(ss_ext_sales_price) DESC, i_brand ASC, i_brand_id ASC, i_manufact_id ASC, i_manufact ASC'
-'| hosts=3 per-host-mem=4.69KB'
-'| tuple-ids=5 row-size=48B cardinality=100'
-'|'
-'17:AGGREGATE [MERGE FINALIZE]'
-'| output: sum(sum(ss_ext_sales_price))'
-'| group by: i_brand, i_brand_id, i_manufact_id, i_manufact'
-'| hosts=3 per-host-mem=10.00MB'
-'| tuple-ids=5 row-size=48B cardinality=87208'
-'|'
-'16:EXCHANGE [HASH(i_brand,i_brand_id,i_manufact_id,i_manufact)]'
-'| hosts=3 per-host-mem=0B'
-'| tuple-ids=5 row-size=48B cardinality=87208'
-'|'
-'09:AGGREGATE'
-'| output: sum(ss_ext_sales_price)'
-'| group by: i_brand, i_brand_id, i_manufact_id, i_manufact'
-'| hosts=3 per-host-mem=10.00MB'
-'| tuple-ids=5 row-size=48B cardinality=87208'
-'|'
-'08:HASH JOIN [INNER JOIN, BROADCAST]'
-'| hash predicates: store_sales.ss_store_sk = store.s_store_sk'
-'| other predicates: substr(ca_zip, 1, 5) != substr(s_zip, 1, 5)'
-'| hosts=3 per-host-mem=383B'
-'| tuple-ids=3,2,0,1,4 row-size=192B cardinality=87208'
-'|'
-'|--15:EXCHANGE [BROADCAST]'
-'| | hosts=3 per-host-mem=0B'
-'| | tuple-ids=4 row-size=29B cardinality=12'
-'| |'
-'| 04:SCAN HDFS [tpcds.store, RANDOM]'
-'| partitions=1/1 size=3.08KB'
-'| table stats: 12 rows total'
-'| column stats: all'
-'| hosts=3 per-host-mem=16.00MB'
-'| tuple-ids=4 row-size=29B cardinality=12'
-'|'
-'07:HASH JOIN [INNER JOIN, BROADCAST]'
-'| hash predicates: store_sales.ss_item_sk = item.i_item_sk'
-'| hosts=3 per-host-mem=14.60KB'
-'| tuple-ids=3,2,0,1 row-size=163B cardinality=87208'
-'|'
-'|--14:EXCHANGE [BROADCAST]'
-'| | hosts=3 per-host-mem=0B'
-'| | tuple-ids=1 row-size=79B cardinality=171'
-'| |'
-'| 01:SCAN HDFS [tpcds.item, RANDOM]'
-'| partitions=1/1 size=4.82MB'
-'| predicates: i_manager_id = 7'
-'| table stats: 18000 rows total'
-'| column stats: all'
-'| hosts=3 per-host-mem=16.00MB'
-'| tuple-ids=1 row-size=79B cardinality=171'
-'|'
-'06:HASH JOIN [INNER JOIN, BROADCAST]'
-'| hash predicates: customer.c_customer_sk = store_sales.ss_customer_sk'
-'| hosts=3 per-host-mem=234.75KB'
-'| tuple-ids=3,2,0 row-size=84B cardinality=87208'
-'|'
-'|--13:EXCHANGE [BROADCAST]'
-'| | hosts=3 per-host-mem=0B'
-'| | tuple-ids=0 row-size=43B cardinality=5082'
-'| |'
-'| 00:SCAN HDFS [tpcds.store_sales, RANDOM]'
-'| partitions=2/120 size=663.52KB'
-'| table stats: 183592 rows total'
-'| column stats: all'
-'| hosts=3 per-host-mem=16.00MB'
-'| tuple-ids=0 row-size=43B cardinality=5082'
-'|'
-'05:HASH JOIN [INNER JOIN, PARTITIONED]'
-'| hash predicates: customer_address.ca_address_sk = customer.c_current_addr_sk'
-'| hosts=3 per-host-mem=429.69KB'
-'| tuple-ids=3,2 row-size=41B cardinality=87208'
-'|'
-'|--12:EXCHANGE [HASH(customer.c_current_addr_sk)]'
-'| | hosts=3 per-host-mem=0B'
-'| | tuple-ids=2 row-size=12B cardinality=100000'
-'| |'
-'| 02:SCAN HDFS [tpcds.customer, RANDOM]'
-'| partitions=1/1 size=12.60MB'
-'| table stats: 100000 rows total'
-'| column stats: all'
-'| hosts=3 per-host-mem=24.00MB'
-'| tuple-ids=2 row-size=12B cardinality=100000'
-'|'
-'11:EXCHANGE [HASH(customer_address.ca_address_sk)]'
-'| hosts=3 per-host-mem=0B'
-'| tuple-ids=3 row-size=29B cardinality=50000'
-'|'
-'03:SCAN HDFS [tpcds.customer_address, RANDOM]'
-' partitions=1/1 size=5.25MB'
-' table stats: 50000 rows total'
+'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
+'PLAN-ROOT SINK'
+'| mem-estimate=0B mem-reservation=0B'
+'|'
+'04:EXCHANGE [UNPARTITIONED]'
+'| mem-estimate=0B mem-reservation=0B'
+'| tuple-ids=0,1 row-size=454B cardinality=5757710'
+'|'
+'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3'
+'02:HASH JOIN [INNER JOIN, BROADCAST]'
+'| hash predicates: l_orderkey = o_orderkey'
+'| runtime filters: RF000 <- o_orderkey'
+'| mem-estimate=300.41MB mem-reservation=136.00MB'
+'| tuple-ids=0,1 row-size=454B cardinality=5757710'
+'|'
+'|--03:EXCHANGE [BROADCAST]'
+'| | mem-estimate=0B mem-reservation=0B'
+'| | tuple-ids=1 row-size=191B cardinality=1500000'
+'| |'
+'| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2'
+'| 01:SCAN HDFS [tpch.orders, RANDOM]'
+row_regex:.*partitions=1/1 files=1 size=.*
+'| table stats: 1500000 rows total'
+'| column stats: all'
+'| mem-estimate=88.00MB mem-reservation=0B'
+'| tuple-ids=1 row-size=191B cardinality=1500000'
+'|'
+'00:SCAN HDFS [tpch.lineitem, RANDOM]'
+row_regex:.*partitions=1/1 files=1 size=.*
+' runtime filters: RF000 -> l_orderkey'
+' table stats: 6001215 rows total'
' column stats: all'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=3 row-size=29B cardinality=50000'
-====
----- QUERY
-# Tests explaining an insert query
-explain insert overwrite functional.alltypessmall (id, string_col)
-partition (year, month)
-select a.id, a.string_col, a.year, a.month from functional.alltypes a
- left semi join functional.alltypesagg b on (a.id = b.id)
-where a.year > 2009 and a.month = 4
-union distinct
-select id, string_col, year, month from functional.alltypes
----- RESULTS
-'Estimated Per-Host Requirements: Memory=30.00MB VCores=3'
-''
-'WRITE TO HDFS [functional.alltypessmall, OVERWRITE=true, PARTITION-KEYS=(year,month)]'
-'| partitions=96'
-'| hosts=3 per-host-mem=69.26KB'
-'|'
-'14:EXCHANGE [HASH(year,month)]'
-'| hosts=3 per-host-mem=0B'
-'| tuple-ids=3 row-size=28B cardinality=7600'
-'|'
-'13:AGGREGATE [MERGE FINALIZE]'
-'| group by: id, string_col, year, month'
-'| hosts=3 per-host-mem=10.00MB'
-'| tuple-ids=3 row-size=28B cardinality=7600'
-'|'
-'12:EXCHANGE [HASH(id,string_col,year,month)]'
-'| hosts=3 per-host-mem=0B'
-'| tuple-ids=3 row-size=28B cardinality=7600'
-'|'
-'|--11:AGGREGATE'
-'| | group by: id, string_col, year, month'
-'| | hosts=3 per-host-mem=10.00MB'
-'| | tuple-ids=3 row-size=28B cardinality=7300'
-'| |'
-'| 10:MERGE'
-'| | hosts=3 per-host-mem=0B'
-'| | tuple-ids=3 row-size=28B cardinality=7300'
-'| |'
-'| 04:SCAN HDFS [functional.alltypes, RANDOM]'
-'| partitions=24/24 size=478.45KB'
-'| table stats: 7300 rows total'
-'| column stats: all'
-'| hosts=3 per-host-mem=16.00MB'
-'| tuple-ids=2 row-size=29B cardinality=7300'
-'|'
-'05:AGGREGATE'
-'| group by: id, string_col, year, month'
-'| hosts=3 per-host-mem=10.00MB'
-'| tuple-ids=3 row-size=28B cardinality=300'
-'|'
-'09:MERGE'
-'| hosts=3 per-host-mem=0B'
-'| tuple-ids=3 row-size=28B cardinality=300'
-'|'
-'03:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]'
-'| hash predicates: a.id = b.id'
-'| hosts=3 per-host-mem=14.32KB'
-'| tuple-ids=0,1 row-size=33B cardinality=300'
-'|'
-'|--07:EXCHANGE [HASH(b.id)]'
-'| | hosts=3 per-host-mem=0B'
-'| | tuple-ids=1 row-size=4B cardinality=10000'
-'| |'
-'| 02:SCAN HDFS [functional.alltypesagg b, RANDOM]'
-'| partitions=10/10 size=743.67KB'
-'| table stats: 10000 rows total'
-'| column stats: all'
-'| hosts=3 per-host-mem=16.00MB'
-'| tuple-ids=1 row-size=4B cardinality=10000'
-'|'
-'06:EXCHANGE [HASH(a.id)]'
-'| hosts=3 per-host-mem=0B'
-'| tuple-ids=0 row-size=29B cardinality=300'
-'|'
-'01:SCAN HDFS [functional.alltypes a, RANDOM]'
-' partitions=1/24 size=19.71KB'
-' table stats: 7300 rows total'
-' column stats: all'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=0 row-size=29B cardinality=300'
-====
----- QUERY
-# Tests explaining an insert query to/from an HBase table
-explain insert into functional_hbase.alltypes
-select a.* from functional_hbase.alltypessmall a
- cross join functional.alltypessmall b
-where a.year > 2009 and a.month = 4
-union all
-select * from functional_hbase.alltypessmall
----- RESULTS
-'Estimated Per-Host Requirements: Memory=1.02GB VCores=3'
-''
-'WRITE TO HBASE table=functional_hbase.alltypes'
-'| hosts=1 per-host-mem=unavailable'
-'|'
-'06:EXCHANGE [UNPARTITIONED]'
-'| hosts=100 per-host-mem=unavailable'
-'| tuple-ids=3 row-size=88B cardinality=596'
-'|'
-'|--08:MERGE'
-'| | hosts=100 per-host-mem=0B'
-'| | tuple-ids=3 row-size=88B cardinality=196'
-'| |'
-'| 04:SCAN HBASE [functional_hbase.alltypessmall]'
-'| table stats: 100 rows total'
-'| column stats: all'
-'| hosts=100 per-host-mem=1.00GB'
-'| tuple-ids=2 row-size=97B cardinality=196'
-'|'
-'07:MERGE'
-'| hosts=100 per-host-mem=0B'
-'| tuple-ids=3 row-size=88B cardinality=400'
-'|'
-'03:CROSS JOIN [BROADCAST]'
-'| hosts=100 per-host-mem=0B'
-'| tuple-ids=0,1 row-size=97B cardinality=400'
-'|'
-'|--05:EXCHANGE [BROADCAST]'
-'| | hosts=3 per-host-mem=0B'
-'| | tuple-ids=1 row-size=0B cardinality=100'
-'| |'
-'| 02:SCAN HDFS [functional.alltypessmall b, RANDOM]'
-'| partitions=4/4 size=6.32KB'
-'| table stats: 100 rows total'
-'| column stats: all'
-'| hosts=3 per-host-mem=16.00MB'
-'| tuple-ids=1 row-size=0B cardinality=100'
-'|'
-'01:SCAN HBASE [functional_hbase.alltypessmall a]'
-' predicates: a.year > 2009, a.month = 4'
-' table stats: 100 rows total'
-' column stats: all'
-' hosts=100 per-host-mem=1.00GB'
-' tuple-ids=0 row-size=97B cardinality=4'
-====
----- QUERY
-# Tests explaining an CTAS statement.
-explain create table t as
-select * from functional.alltypes
-where month = 2
----- RESULTS
-'Estimated Per-Host Requirements: Memory=16.02MB VCores=1'
-''
-'WRITE TO HDFS [functional.t, OVERWRITE=false]'
-'| partitions=1'
-'| hosts=3 per-host-mem=17.62KB'
-'|'
-'00:SCAN HDFS [functional.alltypes, RANDOM]'
-' partitions=2/24 size=36.51KB'
-' table stats: 7300 rows total'
-' column stats: all'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=1 row-size=97B cardinality=560'
+' mem-estimate=88.00MB mem-reservation=0B'
+' tuple-ids=0 row-size=263B cardinality=6001215'
====
---- QUERY
# Tests the warning about missing table stats in the explain header.
@@ -301,63 +50,8 @@ from functional_avro.alltypes t1
inner join functional_parquet.alltypessmall t2 on (t1.id = t2.id)
left outer join functional_avro.alltypes t3 on (t2.id = t3.id)
where t1.month = 1 and t2.year = 2009 and t3.bool_col = false
----- RESULTS
-'Estimated Per-Host Requirements: Memory=4.03GB VCores=3'
+---- RESULTS: VERIFY_IS_SUBSET
+'Per-Host Resource Estimates: Memory=4.03GB'
'WARNING: The following tables are missing relevant table and/or column statistics.'
'functional_avro.alltypes, functional_parquet.alltypessmall'
-''
-'09:AGGREGATE [MERGE FINALIZE]'
-'| output: sum(count(t1.int_col)), sum(sum(t2.float_col)), sum(count(t2.float_col)), sum(sum(t3.bigint_col))'
-'| hosts=3 per-host-mem=unavailable'
-'| tuple-ids=3 row-size=32B cardinality=1'
-'|'
-'08:EXCHANGE [UNPARTITIONED]'
-'| hosts=3 per-host-mem=unavailable'
-'| tuple-ids=3 row-size=32B cardinality=1'
-'|'
-'05:AGGREGATE'
-'| output: count(t1.int_col), sum(t2.float_col), count(t2.float_col), sum(t3.bigint_col)'
-'| hosts=3 per-host-mem=10.00MB'
-'| tuple-ids=3 row-size=32B cardinality=1'
-'|'
-'04:HASH JOIN [LEFT OUTER JOIN, BROADCAST]'
-'| hash predicates: t2.id = t3.id'
-'| other predicates: t3.bool_col = FALSE'
-'| hosts=3 per-host-mem=2.00GB'
-'| tuple-ids=0,1,2N row-size=37B cardinality=unavailable'
-'|'
-'|--07:EXCHANGE [BROADCAST]'
-'| | hosts=3 per-host-mem=0B'
-'| | tuple-ids=2 row-size=13B cardinality=unavailable'
-'| |'
-'| 02:SCAN HDFS [functional_avro.alltypes t3, RANDOM]'
-'| partitions=24/24 size=470.35KB'
-'| predicates: t3.bool_col = FALSE'
-'| table stats: unavailable'
-'| column stats: unavailable'
-'| hosts=3 per-host-mem=16.00MB'
-'| tuple-ids=2 row-size=13B cardinality=unavailable'
-'|'
-'03:HASH JOIN [INNER JOIN, BROADCAST]'
-'| hash predicates: t1.id = t2.id'
-'| hosts=3 per-host-mem=2.00GB'
-'| tuple-ids=0,1 row-size=24B cardinality=unavailable'
-'|'
-'|--06:EXCHANGE [BROADCAST]'
-'| | hosts=3 per-host-mem=0B'
-'| | tuple-ids=1 row-size=12B cardinality=unavailable'
-'| |'
-'| 01:SCAN HDFS [functional_parquet.alltypessmall t2, RANDOM]'
-'| partitions=4/4 size=9.63KB'
-'| table stats: unavailable'
-'| columns missing stats: id, float_col'
-'| hosts=3 per-host-mem=16.00MB'
-'| tuple-ids=1 row-size=12B cardinality=unavailable'
-'|'
-'00:SCAN HDFS [functional_avro.alltypes t1, RANDOM]'
-' partitions=2/24 size=39.87KB'
-' table stats: unavailable'
-' columns missing stats: id, int_col'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=0 row-size=12B cardinality=unavailable'
====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
index 7da9b04..f1ff4a8 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
@@ -1,332 +1,51 @@
====
---- QUERY
-# Tests explaining a query (TPCDS-Q19)
+# Explain a simple hash join query.
explain
-select
- i_brand_id,
- i_brand,
- i_manufact_id,
- i_manufact,
- sum(ss_ext_sales_price) ext_price
-from
- tpcds.store_sales
- join tpcds.item on (store_sales.ss_item_sk = item.i_item_sk)
- join tpcds.customer on (store_sales.ss_customer_sk = customer.c_customer_sk)
- join tpcds.customer_address on (customer.c_current_addr_sk = customer_address.ca_address_sk)
- join tpcds.store on (store_sales.ss_store_sk = store.s_store_sk)
-where
- ss_date between '1999-11-01' and '1999-11-30'
- and i_manager_id = 7
- and substr(ca_zip, 1, 5) <> substr(s_zip, 1, 5)
-group by
- i_brand,
- i_brand_id,
- i_manufact_id,
- i_manufact
-order by
- ext_price desc,
- i_brand,
- i_brand_id,
- i_manufact_id,
- i_manufact
-limit 100
----- RESULTS
-'Estimated Per-Host Requirements: Memory=72.66MB VCores=5'
-''
-'F07:PLAN FRAGMENT [UNPARTITIONED]'
-' 19:TOP-N [LIMIT=100]'
-' | order by: sum(ss_ext_sales_price) DESC, i_brand ASC, i_brand_id ASC, i_manufact_id ASC, i_manufact ASC'
-' | hosts=3 per-host-mem=unavailable'
-' | tuple-ids=5 row-size=48B cardinality=100'
-' |'
-' 18:EXCHANGE [UNPARTITIONED]'
-' hosts=3 per-host-mem=unavailable'
-' tuple-ids=5 row-size=48B cardinality=100'
-''
-'F06:PLAN FRAGMENT [HASH(i_brand,i_brand_id,i_manufact_id,i_manufact)]'
-' DATASTREAM SINK [FRAGMENT=F07, EXCHANGE=18, UNPARTITIONED]'
-' 10:TOP-N [LIMIT=100]'
-' | order by: sum(ss_ext_sales_price) DESC, i_brand ASC, i_brand_id ASC, i_manufact_id ASC, i_manufact ASC'
-' | hosts=3 per-host-mem=4.69KB'
-' | tuple-ids=5 row-size=48B cardinality=100'
-' |'
-' 17:AGGREGATE [MERGE FINALIZE]'
-' | output: sum(sum(ss_ext_sales_price))'
-' | group by: i_brand, i_brand_id, i_manufact_id, i_manufact'
-' | hosts=3 per-host-mem=10.00MB'
-' | tuple-ids=5 row-size=48B cardinality=87208'
-' |'
-' 16:EXCHANGE [HASH(i_brand,i_brand_id,i_manufact_id,i_manufact)]'
-' hosts=3 per-host-mem=0B'
-' tuple-ids=5 row-size=48B cardinality=87208'
-''
-'F02:PLAN FRAGMENT [HASH(customer_address.ca_address_sk)]'
-' DATASTREAM SINK [FRAGMENT=F06, EXCHANGE=16, HASH(i_brand,i_brand_id,i_manufact_id,i_manufact)]'
-' 09:AGGREGATE'
-' | output: sum(ss_ext_sales_price)'
-' | group by: i_brand, i_brand_id, i_manufact_id, i_manufact'
-' | hosts=3 per-host-mem=10.00MB'
-' | tuple-ids=5 row-size=48B cardinality=87208'
-' |'
-' 08:HASH JOIN [INNER JOIN, BROADCAST]'
-' | hash predicates: store_sales.ss_store_sk = store.s_store_sk'
-' | other predicates: substr(ca_zip, 1, 5) != substr(s_zip, 1, 5)'
-' | hosts=3 per-host-mem=383B'
-' | tuple-ids=3,2,0,1,4 row-size=192B cardinality=87208'
-' |'
-' |--15:EXCHANGE [BROADCAST]'
-' | hosts=3 per-host-mem=0B'
-' | tuple-ids=4 row-size=29B cardinality=12'
-' |'
-' 07:HASH JOIN [INNER JOIN, BROADCAST]'
-' | hash predicates: store_sales.ss_item_sk = item.i_item_sk'
-' | hosts=3 per-host-mem=14.60KB'
-' | tuple-ids=3,2,0,1 row-size=163B cardinality=87208'
-' |'
-' |--14:EXCHANGE [BROADCAST]'
-' | hosts=3 per-host-mem=0B'
-' | tuple-ids=1 row-size=79B cardinality=171'
-' |'
-' 06:HASH JOIN [INNER JOIN, BROADCAST]'
-' | hash predicates: customer.c_customer_sk = store_sales.ss_customer_sk'
-' | hosts=3 per-host-mem=234.75KB'
-' | tuple-ids=3,2,0 row-size=84B cardinality=87208'
-' |'
-' |--13:EXCHANGE [BROADCAST]'
-' | hosts=3 per-host-mem=0B'
-' | tuple-ids=0 row-size=43B cardinality=5082'
-' |'
-' 05:HASH JOIN [INNER JOIN, PARTITIONED]'
-' | hash predicates: customer_address.ca_address_sk = customer.c_current_addr_sk'
-' | hosts=3 per-host-mem=429.69KB'
-' | tuple-ids=3,2 row-size=41B cardinality=87208'
-' |'
-' |--12:EXCHANGE [HASH(customer.c_current_addr_sk)]'
-' | hosts=3 per-host-mem=0B'
-' | tuple-ids=2 row-size=12B cardinality=100000'
-' |'
-' 11:EXCHANGE [HASH(customer_address.ca_address_sk)]'
-' hosts=3 per-host-mem=0B'
-' tuple-ids=3 row-size=29B cardinality=50000'
-''
-'F05:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=15, BROADCAST]'
-' 04:SCAN HDFS [tpcds.store, RANDOM]'
-' partitions=1/1 size=3.08KB'
-' table stats: 12 rows total'
-' column stats: all'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=4 row-size=29B cardinality=12'
-''
-'F04:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=14, BROADCAST]'
-' 01:SCAN HDFS [tpcds.item, RANDOM]'
-' partitions=1/1 size=4.82MB'
-' predicates: i_manager_id = 7'
-' table stats: 18000 rows total'
-' column stats: all'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=1 row-size=79B cardinality=171'
-''
-'F03:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=13, BROADCAST]'
-' 00:SCAN HDFS [tpcds.store_sales, RANDOM]'
-' partitions=2/120 size=663.52KB'
-' table stats: 183592 rows total'
-' column stats: all'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=0 row-size=43B cardinality=5082'
-''
-'F01:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=12, HASH(customer.c_current_addr_sk)]'
-' 02:SCAN HDFS [tpcds.customer, RANDOM]'
-' partitions=1/1 size=12.60MB'
-' table stats: 100000 rows total'
-' column stats: all'
-' hosts=3 per-host-mem=24.00MB'
-' tuple-ids=2 row-size=12B cardinality=100000'
-''
-'F00:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=11, HASH(customer_address.ca_address_sk)]'
-' 03:SCAN HDFS [tpcds.customer_address, RANDOM]'
-' partitions=1/1 size=5.25MB'
-' table stats: 50000 rows total'
-' column stats: all'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=3 row-size=29B cardinality=50000'
-====
----- QUERY
-# Tests explaining an insert query
-explain insert overwrite functional.alltypessmall (id, string_col)
-partition (year, month)
-select a.id, a.string_col, a.year, a.month from functional.alltypes a
- left semi join functional.alltypesagg b on (a.id = b.id)
-where a.year > 2009 and a.month = 4
-union distinct
-select id, string_col, year, month from functional.alltypes
----- RESULTS
-'Estimated Per-Host Requirements: Memory=30.00MB VCores=3'
-''
-'F05:PLAN FRAGMENT [HASH(year,month)]'
-' WRITE TO HDFS [functional.alltypessmall, OVERWRITE=true, PARTITION-KEYS=(year,month)]'
-' | partitions=96'
-' | hosts=3 per-host-mem=69.26KB'
-' |'
-' 14:EXCHANGE [HASH(year,month)]'
-' hosts=3 per-host-mem=0B'
-' tuple-ids=3 row-size=28B cardinality=7600'
-''
-'F04:PLAN FRAGMENT [HASH(id,string_col,year,month)]'
-' DATASTREAM SINK [FRAGMENT=F05, EXCHANGE=14, HASH(year,month)]'
-' 13:AGGREGATE [MERGE FINALIZE]'
-' | group by: id, string_col, year, month'
-' | hosts=3 per-host-mem=10.00MB'
-' | tuple-ids=3 row-size=28B cardinality=7600'
-' |'
-' 12:EXCHANGE [HASH(id,string_col,year,month)]'
-' hosts=3 per-host-mem=0B'
-' tuple-ids=3 row-size=28B cardinality=7600'
-''
-'F03:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F04, EXCHANGE=12, HASH(id,string_col,year,month)]'
-' 11:AGGREGATE'
-' | group by: id, string_col, year, month'
-' | hosts=3 per-host-mem=10.00MB'
-' | tuple-ids=3 row-size=28B cardinality=7300'
-' |'
-' 10:MERGE'
-' | hosts=3 per-host-mem=0B'
-' | tuple-ids=3 row-size=28B cardinality=7300'
-' |'
-' 04:SCAN HDFS [functional.alltypes, RANDOM]'
-' partitions=24/24 size=478.45KB'
-' table stats: 7300 rows total'
-' column stats: all'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=2 row-size=29B cardinality=7300'
-''
-'F02:PLAN FRAGMENT [HASH(a.id)]'
-' DATASTREAM SINK [FRAGMENT=F04, EXCHANGE=12, HASH(id,string_col,year,month)]'
-' 05:AGGREGATE'
-' | group by: id, string_col, year, month'
-' | hosts=3 per-host-mem=10.00MB'
-' | tuple-ids=3 row-size=28B cardinality=300'
-' |'
-' 09:MERGE'
-' | hosts=3 per-host-mem=0B'
-' | tuple-ids=3 row-size=28B cardinality=300'
-' |'
-' 03:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]'
-' | hash predicates: a.id = b.id'
-' | hosts=3 per-host-mem=14.32KB'
-' | tuple-ids=0,1 row-size=33B cardinality=300'
-' |'
-' |--07:EXCHANGE [HASH(b.id)]'
-' | hosts=3 per-host-mem=0B'
-' | tuple-ids=1 row-size=4B cardinality=10000'
-' |'
-' 06:EXCHANGE [HASH(a.id)]'
-' hosts=3 per-host-mem=0B'
-' tuple-ids=0 row-size=29B cardinality=300'
-''
-'F01:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=07, HASH(b.id)]'
-' 02:SCAN HDFS [functional.alltypesagg b, RANDOM]'
-' partitions=10/10 size=743.67KB'
-' table stats: 10000 rows total'
-' column stats: all'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=1 row-size=4B cardinality=10000'
-''
-'F00:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=06, HASH(a.id)]'
-' 01:SCAN HDFS [functional.alltypes a, RANDOM]'
-' partitions=1/24 size=19.71KB'
-' table stats: 7300 rows total'
-' column stats: all'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=0 row-size=29B cardinality=300'
-====
----- QUERY
-# Tests explaining an insert query to/from an HBase table
-explain insert into functional_hbase.alltypes
-select a.* from functional_hbase.alltypessmall a
- cross join functional.alltypessmall b
-where a.year > 2009 and a.month = 4
-union all
-select * from functional_hbase.alltypessmall
----- RESULTS
-'Estimated Per-Host Requirements: Memory=1.02GB VCores=3'
-''
-'F03:PLAN FRAGMENT [UNPARTITIONED]'
-' WRITE TO HBASE table=functional_hbase.alltypes'
-' | hosts=1 per-host-mem=unavailable'
-' |'
-' 06:EXCHANGE [UNPARTITIONED]'
-' hosts=100 per-host-mem=unavailable'
-' tuple-ids=3 row-size=88B cardinality=596'
-''
-'F02:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F03, EXCHANGE=06, UNPARTITIONED]'
-' 08:MERGE'
-' | hosts=100 per-host-mem=0B'
-' | tuple-ids=3 row-size=88B cardinality=196'
-' |'
-' 04:SCAN HBASE [functional_hbase.alltypessmall]'
-' table stats: 100 rows total'
-' column stats: all'
-' hosts=100 per-host-mem=1.00GB'
-' tuple-ids=2 row-size=97B cardinality=196'
-''
-'F00:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F03, EXCHANGE=06, UNPARTITIONED]'
-' 07:MERGE'
-' | hosts=100 per-host-mem=0B'
-' | tuple-ids=3 row-size=88B cardinality=400'
-' |'
-' 03:CROSS JOIN [BROADCAST]'
-' | hosts=100 per-host-mem=0B'
-' | tuple-ids=0,1 row-size=97B cardinality=400'
-' |'
-' |--05:EXCHANGE [BROADCAST]'
-' | hosts=3 per-host-mem=0B'
-' | tuple-ids=1 row-size=0B cardinality=100'
-' |'
-' 01:SCAN HBASE [functional_hbase.alltypessmall a]'
-' predicates: a.year > 2009, a.month = 4'
-' table stats: 100 rows total'
-' column stats: all'
-' hosts=100 per-host-mem=1.00GB'
-' tuple-ids=0 row-size=97B cardinality=4'
-''
-'F01:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F00, EXCHANGE=05, BROADCAST]'
-' 02:SCAN HDFS [functional.alltypessmall b, RANDOM]'
-' partitions=4/4 size=6.32KB'
-' table stats: 100 rows total'
-' column stats: all'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=1 row-size=0B cardinality=100'
-====
----- QUERY
-# Tests explaining an CTAS statement.
-explain create table t as
-select * from functional.alltypes
-where month = 2
----- RESULTS
-'Estimated Per-Host Requirements: Memory=16.02MB VCores=1'
-''
-'F00:PLAN FRAGMENT [RANDOM]'
-' WRITE TO HDFS [functional.t, OVERWRITE=false]'
-' | partitions=1'
-' | hosts=3 per-host-mem=17.62KB'
-' |'
-' 00:SCAN HDFS [functional.alltypes, RANDOM]'
-' partitions=2/24 size=36.51KB'
-' table stats: 7300 rows total'
-' column stats: all'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=1 row-size=97B cardinality=560'
+select *
+from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
+---- RESULTS: VERIFY_IS_EQUAL
+'Per-Host Resource Reservation: Memory=136.00MB'
+'Per-Host Resource Estimates: Memory=388.41MB'
+''
+'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
+' PLAN-ROOT SINK'
+' | mem-estimate=0B mem-reservation=0B'
+' |'
+' 04:EXCHANGE [UNPARTITIONED]'
+' mem-estimate=0B mem-reservation=0B'
+' tuple-ids=0,1 row-size=454B cardinality=5757710'
+''
+'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3'
+' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=04, UNPARTITIONED]'
+' | mem-estimate=0B mem-reservation=0B'
+' 02:HASH JOIN [INNER JOIN, BROADCAST]'
+' | hash predicates: l_orderkey = o_orderkey'
+' | runtime filters: RF000 <- o_orderkey'
+' | mem-estimate=300.41MB mem-reservation=136.00MB'
+' | tuple-ids=0,1 row-size=454B cardinality=5757710'
+' |'
+' |--03:EXCHANGE [BROADCAST]'
+' | mem-estimate=0B mem-reservation=0B'
+' | tuple-ids=1 row-size=191B cardinality=1500000'
+' |'
+' 00:SCAN HDFS [tpch.lineitem, RANDOM]'
+row_regex:.*partitions=1/1 files=1 size=.*
+' runtime filters: RF000 -> l_orderkey'
+' table stats: 6001215 rows total'
+' column stats: all'
+' mem-estimate=88.00MB mem-reservation=0B'
+' tuple-ids=0 row-size=263B cardinality=6001215'
+''
+'F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2'
+' DATASTREAM SINK [FRAGMENT=F00, EXCHANGE=03, BROADCAST]'
+' | mem-estimate=0B mem-reservation=0B'
+' 01:SCAN HDFS [tpch.orders, RANDOM]'
+row_regex:.*partitions=1/1 files=1 size=.*
+' table stats: 1500000 rows total'
+' column stats: all'
+' mem-estimate=88.00MB mem-reservation=0B'
+' tuple-ids=1 row-size=191B cardinality=1500000'
====
---- QUERY
# Tests the warning about missing table stats in the explain header.
@@ -335,70 +54,7 @@ from functional_avro.alltypes t1
inner join functional_parquet.alltypessmall t2 on (t1.id = t2.id)
left outer join functional_avro.alltypes t3 on (t2.id = t3.id)
where t1.month = 1 and t2.year = 2009 and t3.bool_col = false
----- RESULTS
-'Estimated Per-Host Requirements: Memory=4.03GB VCores=3'
+---- RESULTS: VERIFY_IS_SUBSET
'WARNING: The following tables are missing relevant table and/or column statistics.'
'functional_avro.alltypes, functional_parquet.alltypessmall'
-''
-'F03:PLAN FRAGMENT [UNPARTITIONED]'
-' 09:AGGREGATE [MERGE FINALIZE]'
-' | output: sum(count(t1.int_col)), sum(sum(t2.float_col)), sum(count(t2.float_col)), sum(sum(t3.bigint_col))'
-' | hosts=3 per-host-mem=unavailable'
-' | tuple-ids=3 row-size=32B cardinality=1'
-' |'
-' 08:EXCHANGE [UNPARTITIONED]'
-' hosts=3 per-host-mem=unavailable'
-' tuple-ids=3 row-size=32B cardinality=1'
-''
-'F00:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F03, EXCHANGE=08, UNPARTITIONED]'
-' 05:AGGREGATE'
-' | output: count(t1.int_col), sum(t2.float_col), count(t2.float_col), sum(t3.bigint_col)'
-' | hosts=3 per-host-mem=10.00MB'
-' | tuple-ids=3 row-size=32B cardinality=1'
-' |'
-' 04:HASH JOIN [LEFT OUTER JOIN, BROADCAST]'
-' | hash predicates: t2.id = t3.id'
-' | other predicates: t3.bool_col = FALSE'
-' | hosts=3 per-host-mem=2.00GB'
-' | tuple-ids=0,1,2N row-size=37B cardinality=unavailable'
-' |'
-' |--07:EXCHANGE [BROADCAST]'
-' | hosts=3 per-host-mem=0B'
-' | tuple-ids=2 row-size=13B cardinality=unavailable'
-' |'
-' 03:HASH JOIN [INNER JOIN, BROADCAST]'
-' | hash predicates: t1.id = t2.id'
-' | hosts=3 per-host-mem=2.00GB'
-' | tuple-ids=0,1 row-size=24B cardinality=unavailable'
-' |'
-' |--06:EXCHANGE [BROADCAST]'
-' | hosts=3 per-host-mem=0B'
-' | tuple-ids=1 row-size=12B cardinality=unavailable'
-' |'
-' 00:SCAN HDFS [functional_avro.alltypes t1, RANDOM]'
-' partitions=2/24 size=39.87KB'
-' table stats: unavailable'
-' columns missing stats: id, int_col'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=0 row-size=12B cardinality=unavailable'
-''
-'F02:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F00, EXCHANGE=07, BROADCAST]'
-' 02:SCAN HDFS [functional_avro.alltypes t3, RANDOM]'
-' partitions=24/24 size=470.35KB'
-' predicates: t3.bool_col = FALSE'
-' table stats: unavailable'
-' column stats: unavailable'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=2 row-size=13B cardinality=unavailable'
-''
-'F01:PLAN FRAGMENT [RANDOM]'
-' DATASTREAM SINK [FRAGMENT=F00, EXCHANGE=06, BROADCAST]'
-' 01:SCAN HDFS [functional_parquet.alltypessmall t2, RANDOM]'
-' partitions=4/4 size=9.63KB'
-' table stats: unavailable'
-' columns missing stats: id, float_col'
-' hosts=3 per-host-mem=16.00MB'
-' tuple-ids=1 row-size=12B cardinality=unavailable'
====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/tests/custom_cluster/test_admission_controller.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index bf677d9..5201c75 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -296,7 +296,8 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
- impalad_args=impalad_admission_ctrl_flags(1, 1, 1234, 1024 * 1024 * 1024),
+ impalad_args=impalad_admission_ctrl_flags(1, 1, 10 * 1024 * 1024,
+ 1024 * 1024 * 1024),
statestored_args=_STATESTORED_ARGS)
def test_trivial_coord_query_limits(self):
"""Tests that trivial coordinator only queries have negligible resource requirements.
@@ -311,7 +312,8 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
"select * from functional.alltypestiny"]
for query in non_trivial_queries:
ex = self.execute_query_expect_failure(self.client, query)
- assert "memory needed 4.00 GB is greater than pool max mem resources" in str(ex)
+ assert re.search("Rejected query from pool default-pool : request memory needed "
+ ".* is greater than pool max mem resources 10.00 MB", str(ex))
class TestAdmissionControllerStress(TestAdmissionControllerBase):
"""Submits a number of queries (parameterized) with some delay between submissions
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/tests/metadata/test_explain.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_explain.py b/tests/metadata/test_explain.py
index fd5b09a..3975e38 100644
--- a/tests/metadata/test_explain.py
+++ b/tests/metadata/test_explain.py
@@ -46,25 +46,29 @@ class TestExplain(ImpalaTestSuite):
v.get_value('exec_option')['disable_codegen'] == False and\
v.get_value('exec_option')['num_nodes'] != 1)
- @pytest.mark.xfail(run=False, reason="Expected per-host mem requirements inconsistent")
+ @pytest.mark.skip_if(pytest.config.option.testing_remote_cluster,
+ reason='Resource profile depends on number of nodes')
def test_explain_level0(self, vector):
vector.get_value('exec_option')['num_scanner_threads'] = self.NUM_SCANNER_THREADS
vector.get_value('exec_option')['explain_level'] = 0
self.run_test_case('QueryTest/explain-level0', vector)
- @pytest.mark.xfail(run=False, reason="Expected per-host mem requirements inconsistent")
+ @pytest.mark.skip_if(pytest.config.option.testing_remote_cluster,
+ reason='Resource profile depends on number of nodes')
def test_explain_level1(self, vector):
vector.get_value('exec_option')['num_scanner_threads'] = self.NUM_SCANNER_THREADS
vector.get_value('exec_option')['explain_level'] = 1
self.run_test_case('QueryTest/explain-level1', vector)
- @pytest.mark.xfail(run=False, reason="The test for missing table stats fails for avro")
+ @pytest.mark.skip_if(pytest.config.option.testing_remote_cluster,
+ reason='Resource profile depends on number of nodes')
def test_explain_level2(self, vector):
vector.get_value('exec_option')['num_scanner_threads'] = self.NUM_SCANNER_THREADS
vector.get_value('exec_option')['explain_level'] = 2
self.run_test_case('QueryTest/explain-level2', vector)
- @pytest.mark.xfail(run=False, reason="The test for missing table stats fails for avro")
+ @pytest.mark.skip_if(pytest.config.option.testing_remote_cluster,
+ reason='Resource profile depends on number of nodes')
def test_explain_level3(self, vector):
vector.get_value('exec_option')['num_scanner_threads'] = self.NUM_SCANNER_THREADS
vector.get_value('exec_option')['explain_level'] = 3