You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2025/09/18 21:03:21 UTC
(impala) branch master updated (591bf48c7 -> ca356a8df)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 591bf48c7 IMPALA-14013: DROP INCREMENTAL STATS throws NullPointerException for Iceberg tables
new dc46aa48d IMPALA-13285: Ignore COMMIT_TXN events on Apache Hive 3
new 3181fe180 IMPALA-13437 (part 1): Compute processing cost before TupleCachePlanner
new ca356a8df IMPALA-13437 (part 2): Implement cost-based tuple cache placement
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/service/query-options.cc | 20 +
be/src/service/query-options.h | 8 +-
be/src/util/backend-gflag-util.cc | 36 +
common/thrift/BackendGflags.thrift | 8 +
common/thrift/ImpalaService.thrift | 25 +
common/thrift/Query.thrift | 20 +
.../org/apache/impala/compat/MetastoreShim.java | 32 +-
.../impala/catalog/events/MetastoreEvents.java | 5 +-
.../java/org/apache/impala/common/TreeNode.java | 2 +-
.../java/org/apache/impala/planner/DataSink.java | 2 +-
.../org/apache/impala/planner/PlanFragment.java | 6 +-
.../java/org/apache/impala/planner/PlanNode.java | 37 +-
.../java/org/apache/impala/planner/Planner.java | 28 +-
.../planner/TupleCacheAllEligiblePolicy.java} | 14 +-
.../impala/planner/TupleCacheCostBasedPolicy.java | 160 ++
.../org/apache/impala/planner/TupleCacheInfo.java | 195 ++-
.../org/apache/impala/planner/TupleCacheNode.java | 18 +-
.../impala/planner/TupleCachePlacementPolicy.java} | 15 +-
.../apache/impala/planner/TupleCachePlanner.java | 72 +-
.../org/apache/impala/service/BackendConfig.java | 16 +
.../java/org/apache/impala/service/Frontend.java | 6 +-
.../catalog/events/EventExecutorServiceTest.java | 8 +
.../events/MetastoreEventsProcessorTest.java | 23 +
.../impala/planner/TpcdsTupleCachePlannerTest.java | 247 +--
.../java/org/apache/impala/testutil/TestUtils.java | 14 +-
.../{tpcds_cpu_cost => tpcds_tuple_cache}/ddl.test | 460 +++++-
.../tpcds-ddl-iceberg.test | 29 +-
.../tpcds-ddl-parquet.test | 29 +-
.../tpcds-q01.test | 144 +-
.../tpcds-q02.test | 174 ++-
.../tpcds-q03.test | 183 ++-
.../tpcds-q04.test | 278 +++-
.../tpcds-q05.test | 299 +++-
.../tpcds-q06.test | 98 +-
.../tpcds-q07.test | 115 +-
.../tpcds-q08.test | 194 ++-
.../tpcds-q09.test | 912 ++++++++++-
.../tpcds-q10a.test | 421 ++++-
.../tpcds-q11.test | 188 ++-
.../tpcds-q12.test | 117 +-
.../tpcds-q13.test | 346 +++-
.../tpcds-q14a.test | 1569 +++++++++++++++++--
.../tpcds-q14b.test | 920 ++++++++++-
.../tpcds-q15.test | 89 +-
.../tpcds-q16.test | 282 +++-
.../tpcds-q17.test | 192 ++-
.../tpcds-q18.test | 193 ++-
.../tpcds-q19.test | 198 ++-
.../tpcds-q20.test | 117 +-
.../tpcds-q21.test | 204 ++-
.../tpcds-q22.test | 125 +-
.../tpcds-q23a.test | 448 +++++-
.../tpcds-q23b.test | 448 +++++-
.../tpcds-q24a.test | 214 ++-
.../tpcds-q24b.test | 214 ++-
.../tpcds-q25.test | 222 ++-
.../tpcds-q26.test | 119 +-
.../tpcds-q27.test | 175 ++-
.../tpcds-q28.test | 402 ++++-
.../tpcds-q29.test | 192 ++-
.../tpcds-q30.test | 246 ++-
.../tpcds-q31.test | 390 ++++-
.../tpcds-q32.test | 213 ++-
.../tpcds-q33.test | 561 ++++++-
.../tpcds-q34.test | 219 ++-
.../tpcds-q35a.test | 195 ++-
.../tpcds-q36.test | 160 +-
.../tpcds-q37.test | 198 ++-
.../tpcds-q38.test | 113 +-
.../tpcds-q39a.test | 158 +-
.../tpcds-q39b.test | 158 +-
.../tpcds-q40.test | 117 +-
.../tpcds-q41.test | 162 +-
.../tpcds-q42.test | 255 ++-
.../tpcds-q43.test | 147 +-
.../tpcds-q44.test | 306 +++-
.../tpcds-q45.test | 156 +-
.../tpcds-q46.test | 143 +-
.../tpcds-q47.test | 143 +-
.../tpcds-q48.test | 265 +++-
.../tpcds-q49.test | 233 ++-
.../tpcds-q50.test | 117 +-
.../tpcds-q51.test | 98 +-
.../tpcds-q52.test | 231 ++-
.../tpcds-q53.test | 292 +++-
.../tpcds-q54.test | 359 ++++-
.../tpcds-q55.test | 255 ++-
.../tpcds-q56.test | 651 +++++++-
.../tpcds-q57.test | 143 +-
.../tpcds-q58.test | 0
.../tpcds-q59.test | 82 +-
.../tpcds-q60.test | 459 +++++-
.../tpcds-q61.test | 441 +++++-
.../tpcds-q62.test | 149 +-
.../tpcds-q63.test | 292 +++-
.../tpcds-q64.test | 177 ++-
.../tpcds-q65.test | 164 +-
.../tpcds-q66.test | 510 +++++-
.../tpcds-q67.test | 94 +-
.../tpcds-q68.test | 143 +-
.../tpcds-q69.test | 273 +++-
.../tpcds-q70.test | 148 +-
.../tpcds-q71.test | 360 ++++-
.../tpcds-q72.test | 233 ++-
.../tpcds-q73.test | 219 ++-
.../tpcds-q74.test | 188 ++-
.../tpcds-q75.test | 502 +++++-
.../tpcds-q76.test | 145 +-
.../tpcds-q77.test | 760 ++++++++-
.../tpcds-q78.test | 97 +-
.../tpcds-q79.test | 143 +-
.../tpcds-q80.test | 384 ++++-
.../tpcds-q81.test | 246 ++-
.../tpcds-q82.test | 160 +-
.../tpcds-q83.test | 555 ++++++-
.../tpcds-q84.test | 120 +-
.../tpcds-q85.test | 309 +++-
.../tpcds-q86.test | 115 +-
.../tpcds-q87.test | 165 +-
.../tpcds-q88.test | 1644 ++++++++++++++++++--
.../tpcds-q89.test | 145 +-
.../tpcds-q90.test | 492 +++++-
.../tpcds-q91.test | 313 +++-
.../tpcds-q92.test | 333 +++-
.../tpcds-q93.test | 67 +-
.../tpcds-q94.test | 282 +++-
.../tpcds-q95.test | 285 +++-
.../tpcds-q96.test | 228 ++-
.../tpcds-q97.test | 98 +-
.../tpcds-q98.test | 111 +-
.../tpcds-q99.test | 149 +-
.../queries/QueryTest/explain-level2.test | 3 +-
tests/custom_cluster/test_tuple_cache.py | 1 +
tests/query_test/test_tuple_cache_tpc_queries.py | 1 +
134 files changed, 27396 insertions(+), 2400 deletions(-)
copy fe/src/{test/java/org/apache/impala/planner/S3Tests.java => main/java/org/apache/impala/planner/TupleCacheAllEligiblePolicy.java} (65%)
create mode 100644 fe/src/main/java/org/apache/impala/planner/TupleCacheCostBasedPolicy.java
copy fe/src/{test/java/org/apache/impala/planner/S3Tests.java => main/java/org/apache/impala/planner/TupleCachePlacementPolicy.java} (68%)
copy java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/TpcdsCpuCostPlannerTest.java => fe/src/test/java/org/apache/impala/planner/TpcdsTupleCachePlannerTest.java (53%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/ddl.test (67%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-ddl-iceberg.test (97%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-ddl-parquet.test (98%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q01.test (87%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q02.test (87%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q03.test (70%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q04.test (91%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q05.test (86%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q06.test (91%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q07.test (85%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q08.test (89%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q09.test (74%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q10a.test (74%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q11.test (91%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q12.test (81%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q13.test (86%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q14a.test (82%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q14b.test (84%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q15.test (86%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q16.test (73%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q17.test (85%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q18.test (90%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q19.test (80%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q20.test (81%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q21.test (75%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q22.test (86%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q23a.test (86%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q23b.test (87%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q24a.test (89%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q24b.test (89%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q25.test (83%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q26.test (84%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q27.test (84%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q28.test (79%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q29.test (85%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q30.test (83%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q31.test (87%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q32.test (76%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q33.test (78%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q34.test (78%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q35a.test (87%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q36.test (86%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q37.test (75%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q38.test (92%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q39a.test (88%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q39b.test (88%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q40.test (86%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q41.test (74%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q42.test (63%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q43.test (80%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q44.test (79%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q45.test (85%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q46.test (88%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q47.test (92%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q48.test (85%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q49.test (89%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q50.test (88%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q51.test (89%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q52.test (65%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q53.test (69%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q54.test (81%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q55.test (62%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q56.test (73%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q57.test (92%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q58.test (100%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q59.test (94%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q60.test (80%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q61.test (78%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q62.test (84%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q63.test (70%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q64.test (96%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q65.test (85%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q66.test (81%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q67.test (94%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q68.test (88%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q69.test (83%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q70.test (90%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q71.test (72%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q72.test (85%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q73.test (77%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q74.test (91%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q75.test (85%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q76.test (87%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q77.test (75%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q78.test (94%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q79.test (84%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q80.test (87%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q81.test (83%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q82.test (80%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q83.test (77%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q84.test (86%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q85.test (86%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q86.test (88%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q87.test (88%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q88.test (71%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q89.test (81%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q90.test (69%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q91.test (75%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q92.test (67%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q93.test (87%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q94.test (73%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q95.test (81%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q96.test (69%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q97.test (87%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q98.test (82%)
copy testdata/workloads/functional-planner/queries/PlannerTest/{tpcds_cpu_cost => tpcds_tuple_cache}/tpcds-q99.test (84%)
(impala) 03/03: IMPALA-13437 (part 2): Implement cost-based tuple cache placement
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit ca356a8df5ea3403910ca460bd709d5fbb801b36
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Mon Jul 21 16:48:57 2025 -0700
IMPALA-13437 (part 2): Implement cost-based tuple cache placement
This changes the default behavior of the tuple cache to consider
cost when placing the TupleCacheNodes. It tries to pick the best
locations within a budget. First, it eliminates unprofitable locations
via a threshold. Next, it ranks the remaining locations by their
profitability. Finally, it picks the best locations in rank order
until it reaches the budget.
The threshold is based on the ratio of processing cost for regular
execution versus the processing cost for reading from the cache.
If the ratio is below the threshold, the location is eliminated.
The threshold is specified by the tuple_cache_required_cost_reduction_factor
query option. This defaults to 3.0, which means that the cost of
reading from the cache must be less than 1/3 the cost of computing
the value normally. A higher value makes this more restrictive
about caching locations, which pushes in the direction of lower
overhead.
The ranking is based on the cost reduction per byte. This is given
by the formula:
(regular processing cost - cost to read from cache) / estimated serialized size
This prefers locations with small results or high reduction in cost.
The budget is based on the estimated serialized size per node. This
limits the total caching that a query will do. A higher value allows more
caching, which can increase the overhead on the first run of a query. A lower
value is less aggressive and can limit the overhead at the expense of less
caching. This uses a per-node limit as the limit should scale based on the
size of the executor group as each executor brings extra capacity. The budget
is specified by the tuple_cache_budget_bytes_per_executor.
The old behavior to place the tuple cache at all eligible locations is
still available via the tuple_cache_placement_policy query option. The
default is the cost_based policy described above, but the old behavior
is available via the all_eligible policy. This is useful for correctness
testing (and the existing tuple cache test cases).
This changes the explain plan output:
- The hash trace is only enabled at VERBOSE level. This means that the regular
profile will not contain the hash trace, as the regular profile uses EXTENDED.
- This adds additional information at VERBOSE to display the cost information
for each plan node. This can help trace why a particular location was
not picked.
Testing:
- This adds a TPC-DS planner test with tuple caching enabled (based on the
existing TpcdsCpuCostPlannerTest)
- This modifies existing tests to adapt to changes in the explain plan output
Change-Id: Ifc6e7b95621a7937d892511dc879bf7c8da07cdc
Reviewed-on: http://gerrit.cloudera.org:8080/23219
Reviewed-by: Michael Smith <mi...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/service/query-options.cc | 20 +
be/src/service/query-options.h | 8 +-
be/src/util/backend-gflag-util.cc | 36 +
common/thrift/BackendGflags.thrift | 8 +
common/thrift/ImpalaService.thrift | 25 +
common/thrift/Query.thrift | 20 +
.../java/org/apache/impala/common/TreeNode.java | 2 +-
.../java/org/apache/impala/planner/PlanNode.java | 32 +-
.../planner/TupleCacheAllEligiblePolicy.java | 31 +
.../impala/planner/TupleCacheCostBasedPolicy.java | 160 +
.../org/apache/impala/planner/TupleCacheInfo.java | 125 +-
.../impala/planner/TupleCachePlacementPolicy.java | 32 +
.../apache/impala/planner/TupleCachePlanner.java | 72 +-
.../org/apache/impala/service/BackendConfig.java | 16 +
.../impala/planner/TpcdsTupleCachePlannerTest.java | 719 ++
.../java/org/apache/impala/testutil/TestUtils.java | 7 +-
.../queries/PlannerTest/tpcds_tuple_cache/ddl.test | 1161 ++++
.../tpcds_tuple_cache/tpcds-ddl-iceberg.test | 796 +++
.../tpcds_tuple_cache/tpcds-ddl-parquet.test | 813 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q01.test | 902 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q02.test | 1000 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q03.test | 524 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q04.test | 2580 +++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q05.test | 1661 +++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q06.test | 948 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q07.test | 643 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q08.test | 998 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q09.test | 2948 ++++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q10a.test | 1376 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q11.test | 1736 +++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q12.test | 530 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q13.test | 954 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q14a.test | 7277 ++++++++++++++++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q14b.test | 4989 ++++++++++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q15.test | 532 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q16.test | 913 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q17.test | 1048 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q18.test | 1020 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q19.test | 847 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q20.test | 526 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q21.test | 639 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q22.test | 548 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q23a.test | 2751 ++++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q23b.test | 3034 ++++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q24a.test | 1624 +++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q24b.test | 1625 +++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q25.test | 1087 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q26.test | 643 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q27.test | 757 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q28.test | 1508 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q29.test | 1058 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q30.test | 1204 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q31.test | 2466 +++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q32.test | 766 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q33.test | 2241 ++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q34.test | 753 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q35a.test | 1185 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q36.test | 715 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q37.test | 649 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q38.test | 1127 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q39a.test | 1035 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q39b.test | 1037 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q40.test | 668 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q41.test | 487 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q42.test | 597 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q43.test | 486 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q44.test | 1369 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q45.test | 831 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q46.test | 897 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q47.test | 1605 +++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q48.test | 794 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q49.test | 1717 +++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q50.test | 693 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q51.test | 814 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q52.test | 573 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q53.test | 779 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q54.test | 1712 +++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q55.test | 589 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q56.test | 2179 ++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q57.test | 1618 +++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q58.test | 1702 +++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q59.test | 995 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q60.test | 2014 ++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q61.test | 1768 +++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q62.test | 671 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q63.test | 796 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q64.test | 4130 +++++++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q65.test | 891 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q66.test | 1727 +++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q67.test | 796 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q68.test | 920 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q69.test | 1341 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q70.test | 1020 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q71.test | 1078 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q72.test | 1329 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q73.test | 750 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q74.test | 1700 +++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q75.test | 2917 ++++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q76.test | 951 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q77.test | 2566 +++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q78.test | 1204 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q79.test | 688 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q80.test | 2346 +++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q81.test | 1204 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q82.test | 618 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q83.test | 2076 ++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q84.test | 709 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q85.test | 1206 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q86.test | 577 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q87.test | 1152 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q88.test | 4844 +++++++++++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q89.test | 634 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q90.test | 1301 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q91.test | 1051 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q92.test | 890 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q93.test | 428 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q94.test | 911 +++
.../PlannerTest/tpcds_tuple_cache/tpcds-q95.test | 1252 ++++
.../PlannerTest/tpcds_tuple_cache/tpcds-q96.test | 630 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q97.test | 582 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q98.test | 527 ++
.../PlannerTest/tpcds_tuple_cache/tpcds-q99.test | 671 ++
.../queries/QueryTest/explain-level2.test | 3 +-
tests/custom_cluster/test_tuple_cache.py | 1 +
tests/query_test/test_tuple_cache_tpc_queries.py | 1 +
125 files changed, 140398 insertions(+), 65 deletions(-)
diff --git a/be/src/service/query-options.cc b/be/src/service/query-options.cc
index 543480f9f..e66c6bc8a 100644
--- a/be/src/service/query-options.cc
+++ b/be/src/service/query-options.cc
@@ -1405,6 +1405,26 @@ Status impala::SetQueryOption(TImpalaQueryOptions::type option, const string& va
query_options->__set_broadcast_cost_scale_factor(double_val);
break;
}
+ case TImpalaQueryOptions::TUPLE_CACHE_PLACEMENT_POLICY: {
+ TTupleCachePlacementPolicy::type enum_type;
+ RETURN_IF_ERROR(GetThriftEnum(value, "Tuple cache placement policy",
+ _TTupleCachePlacementPolicy_VALUES_TO_NAMES, &enum_type));
+ query_options->__set_tuple_cache_placement_policy(enum_type);
+ break;
+ }
+ case TImpalaQueryOptions::TUPLE_CACHE_REQUIRED_COST_REDUCTION_FACTOR: {
+ double double_val = 0.0f;
+ RETURN_IF_ERROR(QueryOptionParser::ParseAndCheckInclusiveLowerBound<double>(
+ option, value, 0.0, &double_val));
+ query_options->__set_tuple_cache_required_cost_reduction_factor(double_val);
+ break;
+ }
+ case TImpalaQueryOptions::TUPLE_CACHE_BUDGET_BYTES_PER_EXECUTOR: {
+ MemSpec mem_spec_val{};
+ RETURN_IF_ERROR(QueryOptionParser::Parse<MemSpec>(option, value, &mem_spec_val));
+ query_options->__set_tuple_cache_budget_bytes_per_executor(mem_spec_val.value);
+ break;
+ }
default:
string key = to_string(option);
if (IsRemovedQueryOption(key)) {
diff --git a/be/src/service/query-options.h b/be/src/service/query-options.h
index 915e598ba..09d0e951c 100644
--- a/be/src/service/query-options.h
+++ b/be/src/service/query-options.h
@@ -51,7 +51,7 @@ typedef std::unordered_map<string, beeswax::TQueryOptionLevel::type>
// plus one. Thus, the second argument to the DCHECK has to be updated every
// time we add or remove a query option to/from the enum TImpalaQueryOptions.
constexpr unsigned NUM_QUERY_OPTIONS =
- TImpalaQueryOptions::BROADCAST_COST_SCALE_FACTOR + 1;
+ TImpalaQueryOptions::TUPLE_CACHE_BUDGET_BYTES_PER_EXECUTOR + 1;
#define QUERY_OPTS_TABLE \
DCHECK_EQ(_TImpalaQueryOptions_VALUES_TO_NAMES.size(), NUM_QUERY_OPTIONS); \
REMOVED_QUERY_OPT_FN(abort_on_default_limit_exceeded, ABORT_ON_DEFAULT_LIMIT_EXCEEDED) \
@@ -380,6 +380,12 @@ constexpr unsigned NUM_QUERY_OPTIONS =
QUERY_OPT_FN(hide_analyzed_query, HIDE_ANALYZED_QUERY, TQueryOptionLevel::ADVANCED) \
QUERY_OPT_FN(broadcast_cost_scale_factor, BROADCAST_COST_SCALE_FACTOR, \
TQueryOptionLevel::ADVANCED) \
+ TUPLE_CACHE_EXEMPT_QUERY_OPT_FN(tuple_cache_placement_policy, \
+ TUPLE_CACHE_PLACEMENT_POLICY, TQueryOptionLevel::ADVANCED) \
+ TUPLE_CACHE_EXEMPT_QUERY_OPT_FN(tuple_cache_required_cost_reduction_factor, \
+ TUPLE_CACHE_REQUIRED_COST_REDUCTION_FACTOR, TQueryOptionLevel::ADVANCED) \
+ TUPLE_CACHE_EXEMPT_QUERY_OPT_FN(tuple_cache_budget_bytes_per_executor, \
+ TUPLE_CACHE_BUDGET_BYTES_PER_EXECUTOR, TQueryOptionLevel::ADVANCED) \
;
/// Enforce practical limits on some query options to avoid undesired query state.
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
index 39dec440f..9f3c1b7a5 100644
--- a/be/src/util/backend-gflag-util.cc
+++ b/be/src/util/backend-gflag-util.cc
@@ -301,6 +301,21 @@ DEFINE_int32(iceberg_catalog_num_threads, 16,
"Maximum number of threads to use for Iceberg catalog operations. These threads are "
"shared among concurrent Iceberg catalog operation (ie., ExpireSnapshot).");
+// These coefficients have not been determined empirically. The write coefficient
+// matches the coefficient for a broadcast sender in DataStreamSink. The read
+// coefficient matches the coefficient for an exchange receiver in ExchandeNode.
+// This includes both a bytes coefficient and a rows coefficient to allow experimentation
+// and tuning.
+// TODO: Tune these empirically.
+DEFINE_double(tuple_cache_cost_coefficient_write_bytes, 0.0027,
+ "Cost coefficient for writing a byte to the tuple cache.");
+DEFINE_double(tuple_cache_cost_coefficient_write_rows, 0.00,
+ "Cost coefficient for writing a row to the tuple cache.");
+DEFINE_double(tuple_cache_cost_coefficient_read_bytes, 0.00,
+ "Cost coefficient for reading a byte from the tuple cache.");
+DEFINE_double(tuple_cache_cost_coefficient_read_rows, 0.1329,
+ "Cost coefficient for reading a row from the tuple cache.");
+
using strings::Substitute;
namespace impala {
@@ -316,6 +331,15 @@ static bool ValidatePositiveDouble(const char* flagname, double value) {
return false;
}
+static bool ValidateNonnegativeDouble(const char* flagname, double value) {
+ if (0.0 <= value) {
+ return true;
+ }
+ LOG(ERROR) << Substitute(
+ "$0 must be greater than or equal to 0.0, value $1 is invalid", flagname, value);
+ return false;
+}
+
static bool ValidatePositiveInt64(const char* flagname, int64_t value) {
if (0 < value) {
return true;
@@ -333,6 +357,10 @@ DEFINE_validator(query_cpu_count_divisor, &ValidatePositiveDouble);
DEFINE_validator(min_processing_per_thread, &ValidatePositiveInt64);
DEFINE_validator(query_cpu_root_factor, &ValidatePositiveDouble);
DEFINE_validator(iceberg_catalog_num_threads, &ValidatePositiveInt32);
+DEFINE_validator(tuple_cache_cost_coefficient_write_bytes, &ValidateNonnegativeDouble);
+DEFINE_validator(tuple_cache_cost_coefficient_write_rows, &ValidateNonnegativeDouble);
+DEFINE_validator(tuple_cache_cost_coefficient_read_bytes, &ValidateNonnegativeDouble);
+DEFINE_validator(tuple_cache_cost_coefficient_read_rows, &ValidateNonnegativeDouble);
Status GetConfigFromCommand(const string& flag_cmd, string& result) {
result.clear();
@@ -552,6 +580,14 @@ Status PopulateThriftBackendGflags(TBackendGflags& cfg) {
cfg.__set_warmup_tables_config_file(FLAGS_warmup_tables_config_file);
cfg.__set_keeps_warmup_tables_loaded(FLAGS_keeps_warmup_tables_loaded);
cfg.__set_truncate_external_tables_with_hms(FLAGS_truncate_external_tables_with_hms);
+ cfg.__set_tuple_cache_cost_coefficient_write_bytes(
+ FLAGS_tuple_cache_cost_coefficient_write_bytes);
+ cfg.__set_tuple_cache_cost_coefficient_write_rows(
+ FLAGS_tuple_cache_cost_coefficient_write_rows);
+ cfg.__set_tuple_cache_cost_coefficient_read_bytes(
+ FLAGS_tuple_cache_cost_coefficient_read_bytes);
+ cfg.__set_tuple_cache_cost_coefficient_read_rows(
+ FLAGS_tuple_cache_cost_coefficient_read_rows);
return Status::OK();
}
diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift
index b186009c8..bd46940a2 100644
--- a/common/thrift/BackendGflags.thrift
+++ b/common/thrift/BackendGflags.thrift
@@ -349,4 +349,12 @@ struct TBackendGflags {
159: required bool keeps_warmup_tables_loaded
160: required bool truncate_external_tables_with_hms
+
+ 161: required double tuple_cache_cost_coefficient_write_bytes
+
+ 162: required double tuple_cache_cost_coefficient_write_rows
+
+ 163: required double tuple_cache_cost_coefficient_read_bytes
+
+ 164: required double tuple_cache_cost_coefficient_read_rows
}
diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift
index 37ed2d957..635816b8c 100644
--- a/common/thrift/ImpalaService.thrift
+++ b/common/thrift/ImpalaService.thrift
@@ -1044,6 +1044,31 @@ enum TImpalaQueryOptions {
// while setting to a value less than 1.0 will favor broadcast joins more.
// Default to 1.0, which means no adjustment is applied.
BROADCAST_COST_SCALE_FACTOR = 194
+
+ // The tuple cache placement policy determines the overarching algorithm for placing
+ // tuple cache locations. The two options are:
+ // all_eligible: Pick all locations that are eligible. This is used for correctness
+ // testing
+ // cost_based: Pick locations that provide cost improvements while imposing a limit
+ // on the total amount of caching to avoid excess overhead.
+ // The default is cost_based.
+ TUPLE_CACHE_PLACEMENT_POLICY = 195
+
+ // Minimum cost reduction necessary for tuple caching to consider a location
+ // A location's cost reduction factor is given by the ratio:
+ // cost of execution / cost of reading from the cache
+ // A higher value for this setting is imposing a higher bar for picking locations,
+ // which means it is caching less aggressively. For example, if this is set to 10,
+ // the cost of reading from the cache needs to be 1/10th the cost of execution to
+ // be considered. A value of zero disables the requirement, which is useful for
+ // correctness checking.
+ TUPLE_CACHE_REQUIRED_COST_REDUCTION_FACTOR = 196
+
+ // Maximum bytes per executor to cache for each query execution. This is using
+ // the estimated serialized size to limit the total number of caching locations
+ // for a given query execution. A higher value caches more aggressively. A lower
+ // value reduces caching and thus overhead.
+ TUPLE_CACHE_BUDGET_BYTES_PER_EXECUTOR = 197
}
// The summary of a DML statement.
diff --git a/common/thrift/Query.thrift b/common/thrift/Query.thrift
index ad99881da..82a094af8 100644
--- a/common/thrift/Query.thrift
+++ b/common/thrift/Query.thrift
@@ -136,6 +136,16 @@ enum TSlotCountStrategy {
PLANNER_CPU_ASK = 1
}
+// Set of options for the tuple_cache_placement_policy. See TupleCachePlanner and
+// associated classes for the specific policies for each option.
+enum TTupleCachePlacementPolicy {
+ // Place tuple cache nodes at all eligible locations
+ ALL_ELIGIBLE = 0,
+
+ // Place tuple cache nodes using costing policies
+ COST_BASED = 1
+}
+
// constants for TQueryOptions.num_nodes
const i32 NUM_NODES_ALL = 0
const i32 NUM_NODES_ALL_RACKS = -1
@@ -788,6 +798,16 @@ struct TQueryOptions {
// See comment in ImpalaService.thrift
195: optional double broadcast_cost_scale_factor = 1.0
+
+ // See comment in ImpalaService.thrift
+ 196: optional TTupleCachePlacementPolicy tuple_cache_placement_policy =
+ TTupleCachePlacementPolicy.COST_BASED;
+
+ // See comment in ImpalaService.thrift
+ 197: optional double tuple_cache_required_cost_reduction_factor = 3.0;
+
+ // See comment in ImpalaService.thrift (defaults to 100MB)
+ 198: optional i64 tuple_cache_budget_bytes_per_executor = 104857600;
}
// Impala currently has three types of sessions: Beeswax, HiveServer2 and external
diff --git a/fe/src/main/java/org/apache/impala/common/TreeNode.java b/fe/src/main/java/org/apache/impala/common/TreeNode.java
index f10465809..08baa9e15 100644
--- a/fe/src/main/java/org/apache/impala/common/TreeNode.java
+++ b/fe/src/main/java/org/apache/impala/common/TreeNode.java
@@ -153,7 +153,7 @@ public abstract class TreeNode<NodeType extends TreeNode<NodeType>> {
*/
@SuppressWarnings("unchecked")
public <C extends TreeNode<NodeType>, D extends C> void collectAll(
- Predicate<? super C> predicate, List<D> matches) {
+ Predicate<? super C> predicate, Collection<D> matches) {
if (predicate.apply((C) this)) matches.add((D) this);
for (NodeType child: children_) child.collectAll(predicate, matches);
}
diff --git a/fe/src/main/java/org/apache/impala/planner/PlanNode.java b/fe/src/main/java/org/apache/impala/planner/PlanNode.java
index acbbbff7f..c45c48481 100644
--- a/fe/src/main/java/org/apache/impala/planner/PlanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/PlanNode.java
@@ -449,35 +449,9 @@ abstract public class PlanNode extends TreeNode<PlanNode> {
}
}
- if (detailLevel.ordinal() >= TExplainLevel.EXTENDED.ordinal()) {
- if (getTupleCacheInfo() != null && getTupleCacheInfo().isEligible()) {
- // This PlanNode is eligible for tuple caching, so there may be TupleCacheNodes
- // above this point. For debuggability, display this node's contribution to the
- // tuple cache key by printing its hash trace.
- //
- // Print trace in chunks to avoid excessive wrapping and padding in impala-shell.
- // There are other explain lines at VERBOSE level that are over 100 chars long so
- // we limit the key chunk length similarly here.
- expBuilder.append(detailPrefix + "tuple cache key: " +
- getTupleCacheInfo().getHashString() + "\n");
- expBuilder.append(detailPrefix + "tuple cache hash trace:\n");
- final int keyFormatWidth = 100;
- for (HashTraceElement elem : getTupleCacheInfo().getHashTraces()) {
- final String hashTrace = elem.getHashTrace();
- if (hashTrace.length() < keyFormatWidth) {
- expBuilder.append(String.format("%s %s: %s\n", detailPrefix,
- elem.getComment(), hashTrace));
- } else {
- expBuilder.append(String.format("%s %s:\n", detailPrefix,
- elem.getComment()));
- for (int idx = 0; idx < hashTrace.length(); idx += keyFormatWidth) {
- int stopIdx = Math.min(hashTrace.length(), idx + keyFormatWidth);
- expBuilder.append(String.format("%s [%s]\n", detailPrefix,
- hashTrace.substring(idx, stopIdx)));
- }
- }
- }
- }
+ // Print information about tuple caching if available
+ if (getTupleCacheInfo() != null) {
+ expBuilder.append(getTupleCacheInfo().getExplainString(detailPrefix, detailLevel));
}
// Print the children. Do not traverse into the children of an Exchange node to
diff --git a/fe/src/main/java/org/apache/impala/planner/TupleCacheAllEligiblePolicy.java b/fe/src/main/java/org/apache/impala/planner/TupleCacheAllEligiblePolicy.java
new file mode 100644
index 000000000..b3ac54290
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/planner/TupleCacheAllEligiblePolicy.java
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.planner;
+
+import java.util.Set;
+
+/**
+ * The all_eligible tuple cache placement policy simply returns all the eligible
+ * locations. This null policy is useful for correctness testing, because we want
+ * to check correctness for all locations without regard for cost.
+ */
+public class TupleCacheAllEligiblePolicy implements TupleCachePlacementPolicy {
+ public Set<PlanNode> getFinalCachingLocations(Set<PlanNode> eligibleLocations) {
+ return eligibleLocations;
+ }
+}
diff --git a/fe/src/main/java/org/apache/impala/planner/TupleCacheCostBasedPolicy.java b/fe/src/main/java/org/apache/impala/planner/TupleCacheCostBasedPolicy.java
new file mode 100644
index 000000000..67148e7b5
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/planner/TupleCacheCostBasedPolicy.java
@@ -0,0 +1,160 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.planner;
+
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.impala.thrift.TQueryOptions;
+
+/**
+ * The cost-based tuple cache placement policy uses the processing cost information
+ * to try to pick the best locations. It has three parts:
+ * 1. Threshold - Locations must meet a certain cost threshold to be considered. The
+ * threshold is currently based on the ratio of the regular processing
+ * cost versus the cost of reading from the cache. This ratio must exceed
+ * the tuple_cache_required_cost_improvement_factor to be considered.
+ * For example, if the tuple_cache_required_cost_improvement_factor is
+ * 3.0, then the cost of reading from the cache must be 1/3rd the cost of
+ * starting from scratch.
+ * 2. Ranking - The locations that meet the threshold are ranked based on the
+ * greatest cost improvement per byte.
+ * 3. Budget - Locations are picked by the ranking order until the budget is reached.
+ * Currently, the budget is based on the number of bytes per executor
+ * set by the tuple_cache_budget_bytes_per_executor.
+ *
+ * The algorithm is fairly flexible, so the specific threshold, ranking, and budget
+ * could be modified over time.
+ */
+public class TupleCacheCostBasedPolicy implements TupleCachePlacementPolicy {
+ private final static Logger LOG =
+ LoggerFactory.getLogger(TupleCacheCostBasedPolicy.class);
+
+ private class CostReductionPerByteComparator implements Comparator<PlanNode> {
+ @Override
+ public int compare(PlanNode n1, PlanNode n2) {
+ Double n1_cost_density = computeCostReductionPerByte(n1);
+ Double n2_cost_density = computeCostReductionPerByte(n2);
+ // To order things such that the highest cost density comes first, we need to flip
+ // the sign on the comparison.
+ int result = -n1_cost_density.compareTo(n2_cost_density);
+ if (result != 0) return result;
+ // Two locations can have the same cost, so this uses the plan node id to break
+ // ties to make it consistent.
+ return n1.getId().asInt() - n2.getId().asInt();
+ }
+ }
+
+ private final Comparator<PlanNode> rankingComparator_;
+ private final TQueryOptions queryOptions_;
+
+ public TupleCacheCostBasedPolicy(TQueryOptions queryOptions) {
+ rankingComparator_ = new CostReductionPerByteComparator();
+ queryOptions_ = queryOptions;
+ }
+
+ private boolean meetsRequiredCostReductionFactor(PlanNode node) {
+ long cumulativeProcessingCost =
+ node.getTupleCacheInfo().getCumulativeProcessingCost();
+ // To avoid division by zero and exotic floating point behavior, require the cache
+ // read processing cost to be 1 or above.
+ long cacheReadProcessingCost =
+ Math.max(node.getTupleCacheInfo().getReadProcessingCost(), 1);
+ double costReductionFactor =
+ (double) cumulativeProcessingCost / cacheReadProcessingCost;
+ double requiredCostReductionFactor =
+ queryOptions_.tuple_cache_required_cost_reduction_factor;
+ if (costReductionFactor < requiredCostReductionFactor) {
+ LOG.trace(String.format("%s eliminated (cost reduction factor %f < threshold %f)",
+ node.getDisplayLabel(), costReductionFactor, requiredCostReductionFactor));
+ return false;
+ }
+ return true;
+ }
+
+ private boolean meetsCostThresholds(PlanNode node) {
+ // Filter out locations without statistics
+ if (node.getTupleCacheInfo().getEstimatedSerializedSize() < 0) {
+ LOG.trace(node.getDisplayLabel() + " eliminated due to missing statistics");
+ return false;
+ }
+ // Filter out locations that exceed the budget. They can never be picked.
+ long budget = queryOptions_.tuple_cache_budget_bytes_per_executor;
+ long bytesPerExecutor = node.getTupleCacheInfo().getEstimatedSerializedSizePerNode();
+ if (bytesPerExecutor > budget) {
+ LOG.trace(String.format("%s eliminated (bytes per executor %d > budget %d)",
+ node.getDisplayLabel(), bytesPerExecutor, budget));
+ return false;
+ }
+ return meetsRequiredCostReductionFactor(node);
+ }
+
+ private Double computeCostReductionPerByte(PlanNode node) {
+ long cumulativeProcessingCost =
+ node.getTupleCacheInfo().getCumulativeProcessingCost();
+ long cacheReadProcessingCost = node.getTupleCacheInfo().getReadProcessingCost();
+ long estimatedSerializedSize = node.getTupleCacheInfo().getEstimatedSerializedSize();
+ long costReduction = cumulativeProcessingCost - cacheReadProcessingCost;
+ // The estimated serialized size can be zero when the cardinality is zero. To keep
+ // this from being infinite, increment the serialized size by one.
+ return Double.valueOf((double) costReduction / (estimatedSerializedSize + 1));
+ }
+
+ public Set<PlanNode> getFinalCachingLocations(Set<PlanNode> eligibleLocations) {
+ Preconditions.checkState(eligibleLocations.size() > 0);
+ PriorityQueue<PlanNode> sortedLocations =
+ new PriorityQueue<PlanNode>(eligibleLocations.size(), rankingComparator_);
+ for (PlanNode node : eligibleLocations) {
+ if (meetsCostThresholds(node)) {
+ sortedLocations.add(node);
+ }
+ }
+ Set<PlanNode> finalLocations = new HashSet<PlanNode>();
+ // We pick the best locations (by the sorting order) until we reach the budget. This
+ // uses the bytes per executor as the units for the budget.
+ long remainingBytesPerExecutorBudget =
+ queryOptions_.tuple_cache_budget_bytes_per_executor;
+ // This continues past a location that would exceed the budget. That allows
+ // smaller locations later in the list to have a chance to fit in the
+ // remaining budget. This also means that one large entry early in the list
+ // won't block any other locations from being considered.
+ while (sortedLocations.size() > 0) {
+ PlanNode node = sortedLocations.poll();
+ long curBytesPerExecutor =
+ node.getTupleCacheInfo().getEstimatedSerializedSizePerNode();
+ if (curBytesPerExecutor > remainingBytesPerExecutorBudget) {
+ LOG.trace(String.format(
+ "Skipped %s (bytes per executor: %d, remaining budget: %d)",
+ node.getDisplayLabel(), curBytesPerExecutor,
+ remainingBytesPerExecutorBudget));
+ continue;
+ }
+ LOG.trace(String.format("Picked %s (bytes per executor: %d, remaining budget: %d)",
+ node.getDisplayLabel(), curBytesPerExecutor, remainingBytesPerExecutorBudget));
+ finalLocations.add(node);
+ remainingBytesPerExecutorBudget -= curBytesPerExecutor;
+ }
+ return finalLocations;
+ }
+}
diff --git a/fe/src/main/java/org/apache/impala/planner/TupleCacheInfo.java b/fe/src/main/java/org/apache/impala/planner/TupleCacheInfo.java
index b3873facc..bcaf81d93 100644
--- a/fe/src/main/java/org/apache/impala/planner/TupleCacheInfo.java
+++ b/fe/src/main/java/org/apache/impala/planner/TupleCacheInfo.java
@@ -36,6 +36,7 @@ import org.apache.impala.catalog.FeView;
import org.apache.impala.common.IdGenerator;
import org.apache.impala.common.PrintUtils;
import org.apache.impala.common.ThriftSerializationCtx;
+import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TExplainLevel;
import org.apache.impala.thrift.TFileSplitGeneratorSpec;
import org.apache.impala.thrift.TScanRange;
@@ -176,6 +177,16 @@ public class TupleCacheInfo {
// the filtered cardinality.
private long estimatedSerializedSize_ = -1;
+ // Estimated size divided by the expected number of nodes. This is used by the cost
+ // based placement for the budget contribution.
+ private long estimatedSerializedSizePerNode_ = -1;
+
+ // Processing cost for writing this location to the cache
+ private long writeProcessingCost_ = -1;
+
+ // Processing cost for reading this location from the cache
+ private long readProcessingCost_ = -1;
+
public TupleCacheInfo(DescriptorTable descTbl) {
ineligibilityReasons_ = EnumSet.noneOf(IneligibilityReason.class);
descriptorTable_ = descTbl;
@@ -205,16 +216,12 @@ public class TupleCacheInfo {
}
public String getHashString() {
- Preconditions.checkState(isEligible(),
- "TupleCacheInfo only has a hash if it is cache eligible");
- Preconditions.checkState(finalized_, "TupleCacheInfo not finalized");
+ checkFinalizedAndEligible("a hash");
return finalizedHashString_;
}
public List<HashTraceElement> getHashTraces() {
- Preconditions.checkState(isEligible(),
- "TupleCacheInfo only has a hash trace if it is cache eligible");
- Preconditions.checkState(finalized_, "TupleCacheInfo not finalized");
+ checkFinalizedAndEligible("a hash trace");
return hashTraces_;
}
@@ -232,17 +239,34 @@ public class TupleCacheInfo {
}
public long getCumulativeProcessingCost() {
- Preconditions.checkState(isEligible(),
- "TupleCacheInfo only has cost information if it is cache eligible.");
- Preconditions.checkState(finalized_, "TupleCacheInfo not finalized");
+ checkFinalizedAndEligible("cost information");
return cumulativeProcessingCost_;
}
public long getEstimatedSerializedSize() {
+ checkFinalizedAndEligible("cost information");
+ return estimatedSerializedSize_;
+ }
+
+ public long getEstimatedSerializedSizePerNode() {
+ checkFinalizedAndEligible("cost information");
+ return estimatedSerializedSizePerNode_;
+ }
+
+ public long getWriteProcessingCost() {
+ checkFinalizedAndEligible("cost information");
+ return writeProcessingCost_;
+ }
+
+ public long getReadProcessingCost() {
+ checkFinalizedAndEligible("cost information");
+ return readProcessingCost_;
+ }
+
+ private void checkFinalizedAndEligible(String contextString) {
Preconditions.checkState(isEligible(),
- "TupleCacheInfo only has cost information if it is cache eligible.");
+ "TupleCacheInfo only has %s if it is cache eligible.", contextString);
Preconditions.checkState(finalized_, "TupleCacheInfo not finalized");
- return estimatedSerializedSize_;
}
/**
@@ -259,6 +283,8 @@ public class TupleCacheInfo {
"TupleCacheInfo only calculates cost information if it is cache eligible.");
Preconditions.checkState(thisPlanNode.getTupleCacheInfo() == this,
"calculateCostInformation() must be called with its enclosing PlanNode");
+ Preconditions.checkState(thisPlanNode.getNumNodes() > 0,
+ "PlanNode fragment must have nodes");
// This was already called on our children, which are known to be eligible.
// Pull in the information from our children.
@@ -280,6 +306,23 @@ public class TupleCacheInfo {
long cardinality = thisPlanNode.getFilteredCardinality();
estimatedSerializedSize_ = (long) Math.round(
ExchangeNode.getAvgSerializedRowSize(thisPlanNode) * cardinality);
+ estimatedSerializedSizePerNode_ =
+ (long) estimatedSerializedSize_ / thisPlanNode.getNumNodes();
+ double costCoefficientWriteBytes =
+ BackendConfig.INSTANCE.getTupleCacheCostCoefficientWriteBytes();
+ double costCoefficientWriteRows =
+ BackendConfig.INSTANCE.getTupleCacheCostCoefficientWriteRows();
+ writeProcessingCost_ =
+ (long) (estimatedSerializedSize_ * costCoefficientWriteBytes +
+ cardinality * costCoefficientWriteRows);
+
+ double costCoefficientReadBytes =
+ BackendConfig.INSTANCE.getTupleCacheCostCoefficientReadBytes();
+ double costCoefficientReadRows =
+ BackendConfig.INSTANCE.getTupleCacheCostCoefficientReadRows();
+ readProcessingCost_ =
+ (long) (estimatedSerializedSize_ * costCoefficientReadBytes +
+ cardinality * costCoefficientReadRows);
}
}
@@ -561,6 +604,53 @@ public class TupleCacheInfo {
return builder.toString();
}
+ public String getExplainHashTrace(String detailPrefix) {
+ StringBuilder output = new StringBuilder();
+ final int keyFormatWidth = 100;
+ for (HashTraceElement elem : getHashTraces()) {
+ final String hashTrace = elem.getHashTrace();
+ if (hashTrace.length() < keyFormatWidth) {
+ output.append(String.format("%s %s: %s\n", detailPrefix, elem.getComment(),
+ hashTrace));
+ } else {
+ output.append(String.format("%s %s:\n", detailPrefix, elem.getComment()));
+ for (int idx = 0; idx < hashTrace.length(); idx += keyFormatWidth) {
+ int stopIdx = Math.min(hashTrace.length(), idx + keyFormatWidth);
+ output.append(String.format("%s [%s]\n", detailPrefix,
+ hashTrace.substring(idx, stopIdx)));
+ }
+ }
+ }
+ return output.toString();
+ }
+
+ public String getExplainString(String detailPrefix, TExplainLevel detailLevel) {
+ if (detailLevel.ordinal() >= TExplainLevel.VERBOSE.ordinal()) {
+ // At extended level, provide information about whether this location is
+ // eligible. If it is, provide the cache key and cost information.
+ StringBuilder output = new StringBuilder();
+ if (isEligible()) {
+ output.append(String.format("%stuple cache key: %s\n", detailPrefix,
+ getHashString()));
+ output.append(getCostExplainString(detailPrefix));
+ // This PlanNode is eligible for tuple caching, so there may be TupleCacheNodes
+ // above this point. For debuggability, display this node's contribution to the
+ // tuple cache key by printing its hash trace.
+ //
+ // Print trace in chunks to avoid excessive wrapping and padding in impala-shell.
+ // There are other explain lines at VERBOSE level that are over 100 chars long so
+ // we limit the key chunk length similarly here.
+ output.append(getExplainHashTrace(detailPrefix));
+ } else {
+ output.append(String.format("%stuple cache ineligibility reasons: %s\n",
+ detailPrefix, getIneligibilityReasonsString()));
+ }
+ return output.toString();
+ } else {
+ return "";
+ }
+ }
+
/**
* Produce explain output describing the cost information for this tuple cache location
*/
@@ -573,9 +663,22 @@ public class TupleCacheInfo {
output.append("unavailable");
}
output.append("\n");
+ output.append(detailPrefix + "estimated serialized size per node: ");
+ if (estimatedSerializedSizePerNode_ > -1) {
+ output.append(PrintUtils.printBytes(estimatedSerializedSizePerNode_));
+ } else {
+ output.append("unavailable");
+ }
+ output.append("\n");
output.append(detailPrefix + "cumulative processing cost: ");
output.append(getCumulativeProcessingCost());
output.append("\n");
+ output.append(detailPrefix + "cache read processing cost: ");
+ output.append(getReadProcessingCost());
+ output.append("\n");
+ output.append(detailPrefix + "cache write processing cost: ");
+ output.append(getWriteProcessingCost());
+ output.append("\n");
return output.toString();
}
diff --git a/fe/src/main/java/org/apache/impala/planner/TupleCachePlacementPolicy.java b/fe/src/main/java/org/apache/impala/planner/TupleCachePlacementPolicy.java
new file mode 100644
index 000000000..a2e7e2866
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/planner/TupleCachePlacementPolicy.java
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.planner;
+
+import java.util.Set;
+
+/**
+ * Shared interface for all tuple cache placement policies.
+ */
+public interface TupleCachePlacementPolicy {
+
+ /**
+ * Given a set of eligible locations, return a set of locations that should actually
+ * be used. The return set must be a subset of the eligible locations passed in.
+ */
+ public Set<PlanNode> getFinalCachingLocations(Set<PlanNode> eligibleLocations);
+}
diff --git a/fe/src/main/java/org/apache/impala/planner/TupleCachePlanner.java b/fe/src/main/java/org/apache/impala/planner/TupleCachePlanner.java
index 2e6d64743..9b2dc9383 100644
--- a/fe/src/main/java/org/apache/impala/planner/TupleCachePlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/TupleCachePlanner.java
@@ -17,10 +17,13 @@
package org.apache.impala.planner;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.thrift.TQueryOptions;
+import org.apache.impala.thrift.TTupleCachePlacementPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,16 +35,34 @@ import com.google.common.base.Preconditions;
* the plan tree be in a stable form that won't later change. That means that this is
* designed to run as the last step in planning.
*
- * The current algorithm is to add a TupleCacheNode at every eligible location. This will
- * need to be refined with cost calculations later.
+ * The cache placement algorithm is controlled by the 'tuple_cache_placement_policy' query
+ * option. See descriptions of these policies at {@link TupleCacheAllEligiblePolicy} and
+ * {@link TupleCacheCostBasedPolicy}.
*/
public class TupleCachePlanner {
private final static Logger LOG = LoggerFactory.getLogger(TupleCachePlanner.class);
private final PlannerContext ctx_;
+ private final TQueryOptions queryOptions_;
+ private final TupleCachePlacementPolicy placementPolicy_;
public TupleCachePlanner(PlannerContext ctx) {
ctx_ = ctx;
+ queryOptions_ =
+ ctx_.getRootAnalyzer().getQueryCtx().client_request.getQuery_options();
+ TTupleCachePlacementPolicy policy = queryOptions_.getTuple_cache_placement_policy();
+ LOG.info("Using tuple cache placement policy: " + policy);
+ switch (policy) {
+ case ALL_ELIGIBLE:
+ placementPolicy_ = new TupleCacheAllEligiblePolicy();
+ break;
+ case COST_BASED:
+ placementPolicy_ = new TupleCacheCostBasedPolicy(queryOptions_);
+ break;
+ default:
+ Preconditions.checkState(false, "Unexpected placement policy: " + policy);
+ placementPolicy_ = null;
+ }
}
/**
@@ -56,8 +77,25 @@ public class TupleCachePlanner {
root.computeTupleCacheInfo(ctx_.getRootAnalyzer().getDescTbl(),
ctx_.getRootAnalyzer().getQueryCtx().query_options_result_hash);
- // Step 2: Build up the new PlanNode tree with TupleCacheNodes added
- PlanNode newRoot = buildCachingPlan(root);
+ // Step 2: Collect eligible locations
+ Set<PlanNode> eligibleLocations = new HashSet<PlanNode>();
+ root.collectAll(
+ (node) -> {
+ return node.getTupleCacheInfo().isEligible() && !node.omitTupleCache();
+ }, eligibleLocations);
+
+ // If there are no eligible locations, we're done
+ if (eligibleLocations.size() == 0) {
+ return plan;
+ }
+
+ // Step 3: Use the placement policy to compute the final locations
+ Set<PlanNode> finalLocations =
+ placementPolicy_.getFinalCachingLocations(eligibleLocations);
+
+ // Step 4: Build up the new PlanNode tree with TupleCacheNodes added in the specified
+ // locations
+ PlanNode newRoot = buildCachingPlan(root, finalLocations);
// Since buildCachingPlan is modifying things in place, verify that the top-most plan
// fragment's plan root matches with the newRoot returned.
Preconditions.checkState(plan.get(0).getPlanRoot() == newRoot);
@@ -71,35 +109,29 @@ public class TupleCachePlanner {
/**
* Add TupleCacheNodes at every eligible location via a bottom-up traversal of the tree.
*/
- private PlanNode buildCachingPlan(PlanNode node) throws ImpalaException {
+ private PlanNode buildCachingPlan(PlanNode node, Set<PlanNode> locations)
+ throws ImpalaException {
// Recurse through the children applying the caching policy
for (int i = 0; i < node.getChildCount(); i++) {
- node.setChild(i, buildCachingPlan(node.getChild(i)));
+ node.setChild(i, buildCachingPlan(node.getChild(i), locations));
}
- // If this node is not eligible, then we are done
- if (!node.getTupleCacheInfo().isEligible()) {
+ // If this node is not in the list of caching locations, then we are done.
+ if (!locations.contains(node)) {
return node;
}
- // If node omits tuple cache placement - such as Exchange and Union nodes, where it
- // would not be beneficial - skip it.
- if (node.omitTupleCache()) {
- return node;
- }
+ // Locations that are not eligible were already filtered out by the collection phase
+ Preconditions.checkState(
+ node.getTupleCacheInfo().isEligible() && !node.omitTupleCache(),
+ "Final location must be eligible");
- // Should we cache above this node?
- // Simplest policy: always cache if eligible
- // TODO: Make this more complicated (e.g. cost calculations)
if (LOG.isTraceEnabled()) {
LOG.trace("Adding TupleCacheNode above node " + node.getId().toString());
}
- // Get current query options
- TQueryOptions queryOptions =
- ctx_.getRootAnalyzer().getQueryCtx().client_request.getQuery_options();
// Allocate TupleCacheNode
TupleCacheNode tupleCacheNode = new TupleCacheNode(ctx_.getNextNodeId(), node,
- queryOptions.isEnable_tuple_cache_verification());
+ queryOptions_.isEnable_tuple_cache_verification());
tupleCacheNode.init(ctx_.getRootAnalyzer());
PlanFragment curFragment = node.getFragment();
if (node == curFragment.getPlanRoot()) {
diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
index a53e8b0cd..65d4fc7e2 100644
--- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
+++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
@@ -597,4 +597,20 @@ public class BackendConfig {
public boolean truncateExternalTablesWithHms() {
return backendCfg_.truncate_external_tables_with_hms;
}
+
+ public double getTupleCacheCostCoefficientWriteBytes() {
+ return backendCfg_.tuple_cache_cost_coefficient_write_bytes;
+ }
+
+ public double getTupleCacheCostCoefficientWriteRows() {
+ return backendCfg_.tuple_cache_cost_coefficient_write_rows;
+ }
+
+ public double getTupleCacheCostCoefficientReadBytes() {
+ return backendCfg_.tuple_cache_cost_coefficient_read_bytes;
+ }
+
+ public double getTupleCacheCostCoefficientReadRows() {
+ return backendCfg_.tuple_cache_cost_coefficient_read_rows;
+ }
}
diff --git a/fe/src/test/java/org/apache/impala/planner/TpcdsTupleCachePlannerTest.java b/fe/src/test/java/org/apache/impala/planner/TpcdsTupleCachePlannerTest.java
new file mode 100644
index 000000000..570730e65
--- /dev/null
+++ b/fe/src/test/java/org/apache/impala/planner/TpcdsTupleCachePlannerTest.java
@@ -0,0 +1,719 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.planner;
+
+import com.google.common.collect.Sets;
+import com.google.common.io.Files;
+
+import org.apache.impala.catalog.SideloadTableStats;
+import org.apache.impala.common.ByteUnits;
+import org.apache.impala.common.RuntimeEnv;
+import org.apache.impala.thrift.TExecutorGroupSet;
+import org.apache.impala.thrift.TQueryOptions;
+import org.apache.impala.thrift.TReplicaPreference;
+import org.apache.impala.thrift.TSlotCountStrategy;
+import org.apache.impala.thrift.TUpdateExecutorMembershipRequest;
+import org.apache.impala.util.ExecutorMembershipSnapshot;
+import org.apache.impala.util.RequestPoolService;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This tests cost-based tuple cache placement for TPC-DS using the same setup as
+ * TpcdsCpuCostPlannerTest, with a simulated 1TB scale and COMPUTE_PROCESSING_COST
+ * option enabled.
+ */
+public class TpcdsTupleCachePlannerTest extends PlannerTestBase {
+ // Pool definitions and includes memory resource limits, copied to a temporary file
+ private static final String ALLOCATION_FILE = "fair-scheduler-3-groups.xml";
+
+ // Contains per-pool configurations for maximum number of running queries and queued
+ // requests.
+ private static final String LLAMA_CONFIG_FILE = "llama-site-3-groups.xml";
+
+ // Planner test option to run each planner test.
+ private static Set<PlannerTestOption> testOptions = tpcdsParquetTestOptions();
+
+ // Query option to run each planner test.
+ private static TQueryOptions options =
+ tpcdsParquetQueryOptions()
+ .setCompute_processing_cost(true)
+ .setMax_fragment_instances_per_node(12)
+ .setReplica_preference(TReplicaPreference.REMOTE)
+ .setSlot_count_strategy(TSlotCountStrategy.PLANNER_CPU_ASK)
+ .setMem_estimate_scale_for_spilling_operator(1.0)
+ .setPlanner_testcase_mode(true)
+ // Required so that output doesn't vary by whether scanned tables have stats &
+ // numRows property or not.
+ .setDisable_hdfs_num_rows_estimate(true)
+ .setEnable_tuple_cache(true);
+
+ // Database name to run this test.
+ private static String testDb = "tpcds_partitioned_parquet_snap";
+
+ // Map of table stats that is obtained through loadStatsJson().
+ private static Map<String, Map<String, SideloadTableStats>> sideloadStats;
+
+ // Granular scan limit that will injected into individual ScanNode of tables.
+ private static Map<String, Long> scanRangeLimit = new HashMap<String, Long>() {
+ {
+ // split a 5752989 bytes file to 10 ranges.
+ put("customer", 580 * ByteUnits.KILOBYTE);
+ // split a 1218792 bytes file to 10 ranges.
+ put("customer_address", 125 * ByteUnits.KILOBYTE);
+ // split a 7848768 bytes file to 10 ranges.
+ put("customer_demographics", 790 * ByteUnits.KILOBYTE);
+ // split a 1815300 bytes file to 4 ranges.
+ put("item", 500L * ByteUnits.KILOBYTE);
+ }
+ };
+
+ // Temporary folder to copy admission control files into.
+ // Do not annotate with JUnit @Rule because we want to keep the tempFolder the same
+ // for entire lifetime of test class.
+ private static TemporaryFolder tempFolder;
+
+ /**
+ * Returns a {@link File} for the file on the classpath.
+ */
+ private static File getClasspathFile(String filename) throws URISyntaxException {
+ return new File(
+ TpcdsCpuCostPlannerTest.class.getClassLoader().getResource(filename).toURI());
+ }
+
+ private static void setupAdmissionControl() throws IOException, URISyntaxException {
+ // Start admission control with config file fair-scheduler-3-groups.xml
+ // and llama-site-3-groups.xml
+ tempFolder = new TemporaryFolder();
+ tempFolder.create();
+ File allocationConfFile = tempFolder.newFile(ALLOCATION_FILE);
+ Files.copy(getClasspathFile(ALLOCATION_FILE), allocationConfFile);
+
+ File llamaConfFile = tempFolder.newFile(LLAMA_CONFIG_FILE);
+ Files.copy(getClasspathFile(LLAMA_CONFIG_FILE), llamaConfFile);
+ // Intentionally mark isTest = false to cache poolService as a singleton.
+ RequestPoolService poolService =
+ RequestPoolService.getInstance(allocationConfFile.getAbsolutePath(),
+ llamaConfFile.getAbsolutePath(), /* isTest */ false);
+ poolService.start();
+ }
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ // Mimic the 10 node test mini-cluster with admission control enabled.
+ setupAdmissionControl();
+ // Add 10 node executor group set root.large. This group set also set with
+ // impala.admission-control.max-query-mem-limit.root.large = 50GB.
+ setUpTestCluster(10, 10, "root.large");
+ setUpKuduClientAndLogDir();
+ Paths.get(outDir_.toString(), "tpcds_tuple_cache").toFile().mkdirs();
+
+ // Sideload stats through RuntimeEnv.
+ RuntimeEnv.INSTANCE.setTestEnv(true);
+ sideloadStats = loadStatsJson("tpcds_cpu_cost/stats-3TB.json");
+ RuntimeEnv.INSTANCE.setSideloadStats(sideloadStats);
+
+ // Artificially split single file table into mutiple scan ranges so that the scan
+ // looks like a multi-files table.
+ for (Map.Entry<String, Long> entry : scanRangeLimit.entrySet()) {
+ RuntimeEnv.INSTANCE.addTableScanRangeLimit(
+ testDb, entry.getKey(), entry.getValue());
+ }
+ invalidateTables();
+ }
+
+ @AfterClass
+ public static void unsetMetadataScaleAndStopPoolService() {
+ RuntimeEnv.INSTANCE.dropSideloadStats();
+ RuntimeEnv.INSTANCE.dropTableScanRangeLimit();
+ invalidateTables();
+
+ RequestPoolService.getInstance().stop();
+ tempFolder.delete();
+ }
+
+ /**
+ * Invalidate tables to reload them with new stats.
+ */
+ private static void invalidateTables() {
+ for (String db : sideloadStats.keySet()) {
+ for (String table : sideloadStats.get(db).keySet()) {
+ catalog_.getSrcCatalog().invalidateTableIfExists(testDb, table);
+ }
+ }
+ }
+
+ @Test
+ public void testQ1() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q01", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ2() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q02", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ3() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q03", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ4() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q04", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ5() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q05", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ6() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q06", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ7() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q07", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ8() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q08", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ9() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q09", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ10() {
+ // This is an official variant of q10 that uses a rewrite for lack of support for
+ // multiple subqueries in disjunctive predicates.
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q10a", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ11() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q11", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ12() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q12", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ13() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q13", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ14a() {
+ // First of the two query statements from the official q14.
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q14a", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ14b() {
+ // Second of the two query statements from the official q14.
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q14b", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ15() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q15", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ16() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q16", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ17() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q17", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ18() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q18", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ19() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q19", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ20() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q20", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ21() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q21", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ22() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q22", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ23a() {
+ // First of the two query statements from the official q23.
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q23a", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ23b() {
+ // Second of the two query statements from the official q23.
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q23b", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ24a() {
+ // First of the two query statements from the official q24.
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q24a", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ24b() {
+ // Second of the two query statements from the official q24.
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q24b", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ25() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q25", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ26() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q26", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ27() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q27", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ28() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q28", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ29() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q29", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ30() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q30", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ31() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q31", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ32() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q32", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ33() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q33", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ34() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q34", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ35() {
+ // This is an official variant of q35 that uses a rewrite for lack of support for
+ // multiple subqueries in disjunctive predicates.
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q35a", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ36() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q36", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ37() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q37", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ38() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q38", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ39a() {
+ // First of the two query statements from the official q39.
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q39a", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ39b() {
+ // Second of the two query statements from the official q39.
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q39b", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ40() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q40", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ41() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q41", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ42() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q42", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ43() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q43", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ44() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q44", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ45() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q45", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ46() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q46", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ47() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q47", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ48() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q48", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ49() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q49", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ50() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q50", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ51() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q51", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ52() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q52", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ53() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q53", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ54() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q54", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ55() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q55", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ56() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q56", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ57() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q57", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ58() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q58", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ59() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q59", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ60() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q60", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ61() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q61", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ62() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q62", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ63() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q63", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ64() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q64", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ65() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q65", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ66() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q66", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ67() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q67", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ68() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q68", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ69() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q69", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ70() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q70", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ71() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q71", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ72() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q72", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ73() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q73", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ74() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q74", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ75() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q75", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ76() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q76", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ77() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q77", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ78() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q78", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ79() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q79", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ80() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q80", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ81() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q81", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ82() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q82", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ83() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q83", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ84() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q84", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ85() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q85", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ86() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q86", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ87() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q87", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ88() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q88", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ89() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q89", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ90() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q90", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ91() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q91", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ92() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q92", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ93() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q93", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ94() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q94", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ95() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q95", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ96() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q96", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ97() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q97", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ98() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q98", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testQ99() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-q99", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testNonTpcdsDdl() {
+ // This is a copy of PlannerTest.testDdl().
+ // Not using tpcds_partitioned_parquet_snap db, but piggy-backed to test them
+ // under costing setup.
+ runPlannerTestFile("tpcds_tuple_cache/ddl", testDb, options, testOptions);
+ }
+
+ @Test
+ public void testTpcdsDdlParquet() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-ddl-parquet", testDb, options,
+ testOptions);
+ }
+
+ @Test
+ public void testTpcdsDdlIceberg() {
+ runPlannerTestFile("tpcds_tuple_cache/tpcds-ddl-iceberg", testDb, options,
+ testOptions);
+ }
+
+}
diff --git a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
index b5036b37a..124ad9571 100644
--- a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
+++ b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
@@ -224,9 +224,14 @@ public class TestUtils {
static IgnoreValueFilter SCAN_RANGE_ROW_COUNT_FILTER =
new IgnoreValueFilter("max-scan-range-rows", PrintUtils.METRIC_REGEX);
+ // Ignore the tuple cache key, as it can vary across different dataloads
+ static IgnoreValueFilter TUPLE_CACHE_KEY_FILTER =
+ new IgnoreValueFilter("cache key", " \\S+", ':');
+
// Filters that are always applied
private static final List<ResultFilter> DEFAULT_FILTERS = Arrays.<ResultFilter>asList(
- SCAN_RANGE_ROW_COUNT_FILTER, new PathFilter("hdfs:"), new PathFilter("file: "));
+ SCAN_RANGE_ROW_COUNT_FILTER, TUPLE_CACHE_KEY_FILTER, new PathFilter("hdfs:"),
+ new PathFilter("file: "));
// Filters that ignore the values of resource requirements that appear in
// "EXTENDED" and above explain plans.
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/ddl.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/ddl.test
new file mode 100644
index 000000000..2f8694809
--- /dev/null
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/ddl.test
@@ -0,0 +1,1161 @@
+# Some of tests here are copied from ../ddl.test
+create table t as select * from functional.alltypes
+---- PLAN
+Max Per-Host Resource Reservation: Memory=4.03MB Threads=1
+Per-Host Resource Estimates: Memory=20MB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=20.10MB mem-reservation=4.03MB thread-reservation=1
+| max-parallelism=1 segment-costs=[3777605]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: functional.alltypes.id, functional.alltypes.bool_col, functional.alltypes.tinyint_col, functional.alltypes.smallint_col, functional.alltypes.int_col, functional.alltypes.bigint_col, functional.alltypes.float_col, functional.alltypes.double_col, functional.alltypes.date_string_col, functional.alltypes.string_col, functional.alltypes.timestamp_col, functional.alltypes.year, functional.alltypes.month
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=3760262
+|
+01:TUPLE CACHE
+| cache key: e9f9ad583e91e3b8b4c9755365a3090c
+| input scan node ids: 0
+| estimated serialized size: 662.99KB
+| estimated serialized size per node: 66.30KB
+| cumulative processing cost: 17343
+| cache read processing cost: 970
+| cache write processing cost: 1833
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypes]
+ HDFS partitions=24/24 files=24 size=478.45KB
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=7.30K size=478.45KB
+ partitions: 24/24 rows=7.30K
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=310
+ mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+ tuple-ids=0 row-size=89B cardinality=7.30K cost=17343
+ in pipelines: 00(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=4.03MB Threads=2
+Per-Host Resource Estimates: Memory=21MB
+F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=1.07MB mem-reservation=0B thread-reservation=1
+| max-parallelism=1 segment-costs=[3764061]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: functional.alltypes.id, functional.alltypes.bool_col, functional.alltypes.tinyint_col, functional.alltypes.smallint_col, functional.alltypes.int_col, functional.alltypes.bigint_col, functional.alltypes.float_col, functional.alltypes.double_col, functional.alltypes.date_string_col, functional.alltypes.string_col, functional.alltypes.timestamp_col, functional.alltypes.year, functional.alltypes.month
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=3760262
+|
+01:EXCHANGE [RANDOM]
+| mem-estimate=993.45KB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=3799
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 24)
+Per-Instance Resources: mem-estimate=20.36MB mem-reservation=4.03MB thread-reservation=1
+max-parallelism=10 segment-costs=[62944]
+02:TUPLE CACHE
+| cache key: e9f9ad583e91e3b8b4c9755365a3090c
+| input scan node ids: 0
+| estimated serialized size: 662.99KB
+| estimated serialized size per node: 66.30KB
+| cumulative processing cost: 17343
+| cache read processing cost: 970
+| cache write processing cost: 1833
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypes, RANDOM]
+ HDFS partitions=24/24 files=24 size=478.45KB
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=7.30K size=478.45KB
+ partitions: 24/24 rows=7.30K
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=310
+ mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+ tuple-ids=0 row-size=89B cardinality=7.30K cost=17343
+ in pipelines: 00(GETNEXT)
+====
+create table t as
+select distinct a.int_col, a.string_col from functional.alltypes a
+inner join functional.alltypessmall b on (a.id = b.id)
+where a.year = 2009 and b.month = 2
+---- PLAN
+Max Per-Host Resource Reservation: Memory=18.98MB Threads=1
+Per-Host Resource Estimates: Memory=51MB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=50.94MB mem-reservation=18.98MB thread-reservation=1 runtime-filters-memory=1.00MB
+| max-parallelism=1 segment-costs=[9748, 3621916]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: a.int_col, a.string_col
+| mem-estimate=221B mem-reservation=0B thread-reservation=0 cost=3621916
+|
+07:TUPLE CACHE
+| cache key: 4e8437e0d41ba7bb619673ef008e7e91
+| input scan node ids: 0,1
+| estimated serialized size: 273B
+| estimated serialized size per node: 27B
+| cumulative processing cost: 9748
+| cache read processing cost: 1
+| cache write processing cost: 0
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=17B cardinality=13 cost=0
+| in pipelines: 03(GETNEXT)
+|
+03:AGGREGATE [FINALIZE]
+| group by: a.int_col, a.string_col
+| mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=2 row-size=17B cardinality=13 cost=76
+| in pipelines: 03(GETNEXT), 00(OPEN)
+|
+06:TUPLE CACHE
+| cache key: 768fd84d79652eb391d1b9a1c58753bb
+| input scan node ids: 0,1
+| estimated serialized size: 429B
+| estimated serialized size per node: 42B
+| cumulative processing cost: 9672
+| cache read processing cost: 1
+| cache write processing cost: 1
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0,1 row-size=25B cardinality=13 cost=0
+| in pipelines: 00(GETNEXT)
+|
+02:HASH JOIN [INNER JOIN]
+| hash predicates: a.id = b.id
+| fk/pk conjuncts: a.id = b.id
+| runtime filters: RF000[bloom] <- b.id
+| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=0,1 row-size=25B cardinality=13 cost=963
+| in pipelines: 00(GETNEXT), 01(OPEN)
+|
+|--05:TUPLE CACHE
+| | cache key: 8ed296fd735ce04ed930f8061b5fbe22
+| | input scan node ids: 1
+| | estimated serialized size: 200B
+| | estimated serialized size per node: 200B
+| | cumulative processing cost: 57
+| | cache read processing cost: 3
+| | cache write processing cost: 0
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=1 row-size=4B cardinality=25 cost=0
+| | in pipelines: 01(GETNEXT)
+| |
+| 01:SCAN HDFS [functional.alltypessmall b]
+| partition predicates: b.`month` = CAST(2 AS INT)
+| HDFS partitions=1/4 files=1 size=1.58KB
+| deterministic scan range assignment: true
+| stored statistics:
+| table: rows=100 size=6.32KB
+| partitions: 1/1 rows=25
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=100
+| mem-estimate=16.00MB mem-reservation=8.00KB thread-reservation=0
+| tuple-ids=1 row-size=4B cardinality=25 cost=57
+| in pipelines: 01(GETNEXT)
+|
+04:TUPLE CACHE
+| cache key: 7bb8e2a858ec75313b864cfe1c4a4e6f
+| input scan node ids: 0
+| estimated serialized size: 89.11KB
+| estimated serialized size per node: 8.91KB
+| cumulative processing cost: 8652
+| cache read processing cost: 485
+| cache write processing cost: 246
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=21B cardinality=3.65K cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypes a]
+ partition predicates: a.`year` = CAST(2009 AS INT)
+ HDFS partitions=12/24 files=12 size=238.68KB
+ deterministic scan range assignment: true
+ runtime filters: RF000[bloom] -> a.id
+ stored statistics:
+ table: rows=7.30K size=478.45KB
+ partitions: 12/12 rows=3.65K
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=622
+ mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+ tuple-ids=0 row-size=21B cardinality=3.65K cost=8652
+ in pipelines: 00(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=23.91MB Threads=4
+Per-Host Resource Estimates: Memory=72MB
+F02:PLAN FRAGMENT [HASH(a.int_col,a.string_col)] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=10.02MB mem-reservation=1.94MB thread-reservation=1
+| max-parallelism=1 segment-costs=[78, 3621916] cpu-comparison-result=11 [max(1 (self) vs 11 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: a.int_col, a.string_col
+| mem-estimate=221B mem-reservation=0B thread-reservation=0 cost=3621916
+|
+06:AGGREGATE [FINALIZE]
+| group by: a.int_col, a.string_col
+| mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=2 row-size=17B cardinality=13 cost=76
+| in pipelines: 06(GETNEXT), 00(OPEN)
+|
+05:EXCHANGE [HASH(a.int_col,a.string_col)]
+| mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| tuple-ids=2 row-size=17B cardinality=13 cost=2
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 12)
+Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+Per-Instance Resources: mem-estimate=38.08MB mem-reservation=14.03MB thread-reservation=1
+max-parallelism=10 segment-costs=[9666, 20] cpu-comparison-result=11 [max(10 (self) vs 11 (sum children))]
+10:TUPLE CACHE
+| cache key: 8299471539153599d0f44404bd572e63
+| input scan node ids: 0
+| estimated serialized size: 273B
+| estimated serialized size per node: 27B
+| cumulative processing cost: 9778
+| cache read processing cost: 1
+| cache write processing cost: 0
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=17B cardinality=13 cost=0
+| in pipelines: 00(GETNEXT)
+|
+03:AGGREGATE [STREAMING]
+| group by: a.int_col, a.string_col
+| mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=2 row-size=17B cardinality=13 cost=76
+| in pipelines: 00(GETNEXT)
+|
+09:TUPLE CACHE
+| cache key: 6d91c292f1c154c2042d1605389c0440
+| input scan node ids: 0
+| estimated serialized size: 429B
+| estimated serialized size per node: 42B
+| cumulative processing cost: 9702
+| cache read processing cost: 1
+| cache write processing cost: 1
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0,1 row-size=25B cardinality=13 cost=0
+| in pipelines: 00(GETNEXT)
+|
+02:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=00
+| hash predicates: a.id = b.id
+| fk/pk conjuncts: a.id = b.id
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=0,1 row-size=25B cardinality=13 cost=938
+| in pipelines: 00(GETNEXT), 01(OPEN)
+|
+|--F03:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=2.95MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[55]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: b.id
+| | runtime filters: RF000[bloom] <- b.id
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0 cost=25
+| |
+| 04:EXCHANGE [BROADCAST]
+| | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=1 row-size=4B cardinality=25 cost=30
+| | in pipelines: 01(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=20.03MB mem-reservation=4.01MB thread-reservation=1
+| max-parallelism=1 segment-costs=[57]
+| 08:TUPLE CACHE
+| | cache key: 8ed296fd735ce04ed930f8061b5fbe22
+| | input scan node ids: 1
+| | estimated serialized size: 200B
+| | estimated serialized size per node: 200B
+| | cumulative processing cost: 57
+| | cache read processing cost: 3
+| | cache write processing cost: 0
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=1 row-size=4B cardinality=25 cost=0
+| | in pipelines: 01(GETNEXT)
+| |
+| 01:SCAN HDFS [functional.alltypessmall b, RANDOM]
+| partition predicates: b.`month` = CAST(2 AS INT)
+| HDFS partitions=1/4 files=1 size=1.58KB
+| deterministic scan range assignment: true
+| stored statistics:
+| table: rows=100 size=6.32KB
+| partitions: 1/1 rows=25
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=100
+| mem-estimate=16.00MB mem-reservation=8.00KB thread-reservation=0
+| tuple-ids=1 row-size=4B cardinality=25 cost=57
+| in pipelines: 01(GETNEXT)
+|
+07:TUPLE CACHE
+| cache key: 6a50d18a98795cc3f86b2abe97b121a2
+| input scan node ids: 0
+| estimated serialized size: 89.11KB
+| estimated serialized size per node: 8.91KB
+| cumulative processing cost: 8652
+| cache read processing cost: 485
+| cache write processing cost: 246
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=21B cardinality=3.65K cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypes a, RANDOM]
+ partition predicates: a.`year` = CAST(2009 AS INT)
+ HDFS partitions=12/24 files=12 size=238.68KB
+ deterministic scan range assignment: true
+ runtime filters: RF000[bloom] -> a.id
+ stored statistics:
+ table: rows=7.30K size=478.45KB
+ partitions: 12/12 rows=3.65K
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=622
+ mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+ tuple-ids=0 row-size=21B cardinality=3.65K cost=8652
+ in pipelines: 00(GETNEXT)
+====
+# CTAS with a view that has a limit clause (IMPALA-1411)
+create table t as
+select int_col from functional.hive_view
+---- PLAN
+Max Per-Host Resource Reservation: Memory=32.00KB Threads=1
+Per-Host Resource Estimates: Memory=16MB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=1
+| max-parallelism=1 segment-costs=[3639241]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: CAST(1 AS TINYINT)
+| mem-estimate=0B mem-reservation=0B thread-reservation=0 cost=3621898
+|
+00:SCAN HDFS [functional.alltypes]
+ HDFS partitions=24/24 files=24 size=478.45KB
+ stored statistics:
+ table: rows=7.30K size=478.45KB
+ partitions: 24/24 rows=7.30K
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=310
+ limit: 1
+ mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+ tuple-ids=0 row-size=0B cardinality=1 cost=17343
+ in pipelines: 00(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=32.00KB Threads=2
+Per-Host Resource Estimates: Memory=16MB
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=16.00KB mem-reservation=0B thread-reservation=1
+| max-parallelism=1 segment-costs=[3621898]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: CAST(1 AS TINYINT)
+| mem-estimate=0B mem-reservation=0B thread-reservation=0 cost=3621898
+|
+01:EXCHANGE [UNPARTITIONED]
+| limit: 1
+| mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=0B cardinality=1 cost=0
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 24)
+Per-Instance Resources: mem-estimate=16.02MB mem-reservation=32.00KB thread-reservation=1
+max-parallelism=10 segment-costs=[17343]
+00:SCAN HDFS [functional.alltypes, RANDOM]
+ HDFS partitions=24/24 files=24 size=478.45KB
+ stored statistics:
+ table: rows=7.30K size=478.45KB
+ partitions: 24/24 rows=7.30K
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=310
+ limit: 1
+ mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+ tuple-ids=0 row-size=0B cardinality=1 cost=17343
+ in pipelines: 00(GETNEXT)
+====
+# CTAS with multiple nested inline views that have a limit clause (IMPALA-1411)
+create table t as
+select * from
+ (select * from
+ (select int_col from functional.alltypestiny limit 1) v1 ) v2
+---- PLAN
+Max Per-Host Resource Reservation: Memory=8.00KB Threads=1
+Per-Host Resource Estimates: Memory=16MB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=16.00MB mem-reservation=8.00KB thread-reservation=1
+| max-parallelism=1 segment-costs=[3621915]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: int_col
+| mem-estimate=4B mem-reservation=0B thread-reservation=0 cost=3621899
+|
+00:SCAN HDFS [functional.alltypestiny]
+ HDFS partitions=4/4 files=4 size=460B
+ stored statistics:
+ table: rows=8 size=460B
+ partitions: 4/4 rows=8
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=2
+ limit: 1
+ mem-estimate=16.00MB mem-reservation=8.00KB thread-reservation=0
+ tuple-ids=0 row-size=4B cardinality=1 cost=16
+ in pipelines: 00(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=8.00KB Threads=2
+Per-Host Resource Estimates: Memory=16MB
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=16.00KB mem-reservation=0B thread-reservation=1
+| max-parallelism=1 segment-costs=[3621899]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: int_col
+| mem-estimate=4B mem-reservation=0B thread-reservation=0 cost=3621899
+|
+01:EXCHANGE [UNPARTITIONED]
+| limit: 1
+| mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=4B cardinality=1 cost=0
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+Per-Instance Resources: mem-estimate=16.03MB mem-reservation=8.00KB thread-reservation=1
+max-parallelism=4 segment-costs=[16]
+00:SCAN HDFS [functional.alltypestiny, RANDOM]
+ HDFS partitions=4/4 files=4 size=460B
+ stored statistics:
+ table: rows=8 size=460B
+ partitions: 4/4 rows=8
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=2
+ limit: 1
+ mem-estimate=16.00MB mem-reservation=8.00KB thread-reservation=0
+ tuple-ids=0 row-size=4B cardinality=1 cost=16
+ in pipelines: 00(GETNEXT)
+====
+
+# CTAS with a select statement that has a limit and offset clause (IMPALA-1411)
+create table t as
+select * from functional.alltypestiny order by id limit 1 offset 5
+---- PLAN
+Max Per-Host Resource Reservation: Memory=4.01MB Threads=1
+Per-Host Resource Estimates: Memory=20MB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=20.00MB mem-reservation=4.01MB thread-reservation=1
+| max-parallelism=1 segment-costs=[29, 3621916]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month
+| mem-estimate=89B mem-reservation=0B thread-reservation=0 cost=3621916
+|
+01:TOP-N [LIMIT=1 OFFSET=5]
+| order by: id ASC
+| mem-estimate=534B mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=89B cardinality=1 cost=13
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+02:TUPLE CACHE
+| cache key: 7663eaa15083f0de0bc5722b03d246d9
+| input scan node ids: 0
+| estimated serialized size: 744B
+| estimated serialized size per node: 186B
+| cumulative processing cost: 16
+| cache read processing cost: 1
+| cache write processing cost: 2
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=8 cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypestiny]
+ HDFS partitions=4/4 files=4 size=460B
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=8 size=460B
+ partitions: 4/4 rows=8
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=2
+ mem-estimate=16.00MB mem-reservation=8.00KB thread-reservation=0
+ tuple-ids=0 row-size=89B cardinality=8 cost=16
+ in pipelines: 00(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=4.01MB Threads=2
+Per-Host Resource Estimates: Memory=20MB
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=372.09KB mem-reservation=0B thread-reservation=1
+| max-parallelism=1 segment-costs=[3621900] cpu-comparison-result=4 [max(1 (self) vs 4 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month
+| mem-estimate=89B mem-reservation=0B thread-reservation=0 cost=3621898
+|
+02:MERGING-EXCHANGE [UNPARTITIONED]
+| offset: 5
+| order by: id ASC
+| limit: 1
+| mem-estimate=372.00KB mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=89B cardinality=0 cost=2
+| in pipelines: 01(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+Per-Instance Resources: mem-estimate=20.00MB mem-reservation=4.01MB thread-reservation=1
+max-parallelism=4 segment-costs=[29, 0]
+01:TOP-N [LIMIT=6]
+| order by: id ASC
+| mem-estimate=534B mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=89B cardinality=6 cost=13
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+03:TUPLE CACHE
+| cache key: 7663eaa15083f0de0bc5722b03d246d9
+| input scan node ids: 0
+| estimated serialized size: 744B
+| estimated serialized size per node: 186B
+| cumulative processing cost: 16
+| cache read processing cost: 1
+| cache write processing cost: 2
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=8 cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypestiny, RANDOM]
+ HDFS partitions=4/4 files=4 size=460B
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=8 size=460B
+ partitions: 4/4 rows=8
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=2
+ mem-estimate=16.00MB mem-reservation=8.00KB thread-reservation=0
+ tuple-ids=0 row-size=89B cardinality=8 cost=16
+ in pipelines: 00(GETNEXT)
+====
+# CTAS with an inline view that has a limit and offset clause (IMPALA-1411)
+create table t as
+select * from
+ (select int_col from functional.alltypestiny order by id limit 2 offset 5) v
+---- PLAN
+Max Per-Host Resource Reservation: Memory=4.01MB Threads=1
+Per-Host Resource Estimates: Memory=20MB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=20.00MB mem-reservation=4.01MB thread-reservation=1
+| max-parallelism=1 segment-costs=[29, 3621902]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: int_col
+| mem-estimate=16B mem-reservation=0B thread-reservation=0 cost=3621902
+|
+01:TOP-N [LIMIT=2 OFFSET=5]
+| order by: id ASC
+| mem-estimate=56B mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=8B cardinality=2 cost=13
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+02:TUPLE CACHE
+| cache key: 7d02972699db783a7b0ff21ac027c89c
+| input scan node ids: 0
+| estimated serialized size: 96B
+| estimated serialized size per node: 24B
+| cumulative processing cost: 16
+| cache read processing cost: 1
+| cache write processing cost: 0
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=8B cardinality=8 cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypestiny]
+ HDFS partitions=4/4 files=4 size=460B
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=8 size=460B
+ partitions: 4/4 rows=8
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=2
+ mem-estimate=16.00MB mem-reservation=8.00KB thread-reservation=0
+ tuple-ids=0 row-size=8B cardinality=8 cost=16
+ in pipelines: 00(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=4.01MB Threads=2
+Per-Host Resource Estimates: Memory=20MB
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=48.01KB mem-reservation=0B thread-reservation=1
+| max-parallelism=1 segment-costs=[3621899] cpu-comparison-result=4 [max(1 (self) vs 4 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: int_col
+| mem-estimate=8B mem-reservation=0B thread-reservation=0 cost=3621898
+|
+02:MERGING-EXCHANGE [UNPARTITIONED]
+| offset: 5
+| order by: id ASC
+| limit: 2
+| mem-estimate=48.00KB mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=8B cardinality=0 cost=1
+| in pipelines: 01(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+Per-Instance Resources: mem-estimate=20.00MB mem-reservation=4.01MB thread-reservation=1
+max-parallelism=4 segment-costs=[29, 0]
+01:TOP-N [LIMIT=7]
+| order by: id ASC
+| mem-estimate=56B mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=8B cardinality=7 cost=13
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+03:TUPLE CACHE
+| cache key: 7d02972699db783a7b0ff21ac027c89c
+| input scan node ids: 0
+| estimated serialized size: 96B
+| estimated serialized size per node: 24B
+| cumulative processing cost: 16
+| cache read processing cost: 1
+| cache write processing cost: 0
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=8B cardinality=8 cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypestiny, RANDOM]
+ HDFS partitions=4/4 files=4 size=460B
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=8 size=460B
+ partitions: 4/4 rows=8
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=2
+ mem-estimate=16.00MB mem-reservation=8.00KB thread-reservation=0
+ tuple-ids=0 row-size=8B cardinality=8 cost=16
+ in pipelines: 00(GETNEXT)
+====
+# CTAS with sort columns
+create table t sort by (int_col, bool_col) as
+select * from functional.alltypes
+---- PLAN
+Max Per-Host Resource Reservation: Memory=16.03MB Threads=1
+Per-Host Resource Estimates: Memory=32MB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=32.00MB mem-reservation=16.03MB thread-reservation=1
+| max-parallelism=1 segment-costs=[48576, 3760262]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=3760262
+|
+01:SORT
+| order by: int_col ASC NULLS LAST, bool_col ASC NULLS LAST
+| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=89B cardinality=7.30K cost=31233
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+02:TUPLE CACHE
+| cache key: e9f9ad583e91e3b8b4c9755365a3090c
+| input scan node ids: 0
+| estimated serialized size: 662.99KB
+| estimated serialized size per node: 66.30KB
+| cumulative processing cost: 17343
+| cache read processing cost: 970
+| cache write processing cost: 1833
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypes]
+ HDFS partitions=24/24 files=24 size=478.45KB
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=7.30K size=478.45KB
+ partitions: 24/24 rows=7.30K
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=310
+ mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+ tuple-ids=0 row-size=89B cardinality=7.30K cost=17343
+ in pipelines: 00(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=16.03MB Threads=2
+Per-Host Resource Estimates: Memory=33MB
+F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=12.97MB mem-reservation=12.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[35032, 3760262]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false]
+| partitions=1
+| output exprs: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=3760262
+|
+02:SORT
+| order by: int_col ASC NULLS LAST, bool_col ASC NULLS LAST
+| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=89B cardinality=7.30K cost=31233
+| in pipelines: 02(GETNEXT), 00(OPEN)
+|
+01:EXCHANGE [RANDOM]
+| mem-estimate=993.45KB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=3799
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 24)
+Per-Instance Resources: mem-estimate=20.36MB mem-reservation=4.03MB thread-reservation=1
+max-parallelism=10 segment-costs=[62944]
+03:TUPLE CACHE
+| cache key: e9f9ad583e91e3b8b4c9755365a3090c
+| input scan node ids: 0
+| estimated serialized size: 662.99KB
+| estimated serialized size per node: 66.30KB
+| cumulative processing cost: 17343
+| cache read processing cost: 970
+| cache write processing cost: 1833
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypes, RANDOM]
+ HDFS partitions=24/24 files=24 size=478.45KB
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=7.30K size=478.45KB
+ partitions: 24/24 rows=7.30K
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=310
+ mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+ tuple-ids=0 row-size=89B cardinality=7.30K cost=17343
+ in pipelines: 00(GETNEXT)
+====
+# CTAS with partitions and sort columns
+create table t partitioned by (year, month) sort by (int_col, bool_col) as
+select * from functional.alltypes
+---- PLAN
+Max Per-Host Resource Reservation: Memory=16.03MB Threads=1
+Per-Host Resource Estimates: Memory=32MB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=32.00MB mem-reservation=16.03MB thread-reservation=1
+| max-parallelism=1 segment-costs=[48576, 3760262]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false, PARTITION-KEYS=(year,month)]
+| partitions=24
+| output exprs: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=3760262
+|
+01:SORT
+| order by: year ASC NULLS LAST, month ASC NULLS LAST, int_col ASC NULLS LAST, bool_col ASC NULLS LAST
+| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=89B cardinality=7.30K cost=31233
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+02:TUPLE CACHE
+| cache key: e9f9ad583e91e3b8b4c9755365a3090c
+| input scan node ids: 0
+| estimated serialized size: 662.99KB
+| estimated serialized size per node: 66.30KB
+| cumulative processing cost: 17343
+| cache read processing cost: 970
+| cache write processing cost: 1833
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypes]
+ HDFS partitions=24/24 files=24 size=478.45KB
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=7.30K size=478.45KB
+ partitions: 24/24 rows=7.30K
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=310
+ mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+ tuple-ids=0 row-size=89B cardinality=7.30K cost=17343
+ in pipelines: 00(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=16.03MB Threads=2
+Per-Host Resource Estimates: Memory=37MB
+F01:PLAN FRAGMENT [HASH(functional.alltypes.year,functional.alltypes.month)] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=12.97MB mem-reservation=12.00MB thread-reservation=1
+| max-parallelism=10 segment-costs=[35032, 3760262]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false, PARTITION-KEYS=(year,month)]
+| partitions=24
+| output exprs: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month
+| mem-estimate=63.45KB mem-reservation=0B thread-reservation=0 cost=3760262
+|
+02:SORT
+| order by: year ASC NULLS LAST, month ASC NULLS LAST, int_col ASC NULLS LAST, bool_col ASC NULLS LAST
+| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=89B cardinality=7.30K cost=31233
+| in pipelines: 02(GETNEXT), 00(OPEN)
+|
+01:EXCHANGE [HASH(functional.alltypes.year,functional.alltypes.month)]
+| mem-estimate=993.45KB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=3799
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 24)
+Per-Instance Resources: mem-estimate=23.63MB mem-reservation=4.03MB thread-reservation=1
+max-parallelism=10 segment-costs=[62944]
+03:TUPLE CACHE
+| cache key: e9f9ad583e91e3b8b4c9755365a3090c
+| input scan node ids: 0
+| estimated serialized size: 662.99KB
+| estimated serialized size per node: 66.30KB
+| cumulative processing cost: 17343
+| cache read processing cost: 970
+| cache write processing cost: 1833
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypes, RANDOM]
+ HDFS partitions=24/24 files=24 size=478.45KB
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=7.30K size=478.45KB
+ partitions: 24/24 rows=7.30K
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=310
+ mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+ tuple-ids=0 row-size=89B cardinality=7.30K cost=17343
+ in pipelines: 00(GETNEXT)
+====
+# IMPALA-4167: if no (no)shuffle hint is given for CTAS into partitioned HDFS table, then
+# Impala is free to decide whether to add an exchange node or not. In this example, the
+# planner decides to shuffle. The aim of this test is to have a reference query where
+# noshuffle hint would lead to a different plan.
+# Note that noclustered hint is added to ensure consistent plans on Impala 2.x and 3.x,
+# because IMPALA-5293 changed clustered to be the default on 3.x.
+create /*+ noclustered */table t partitioned by (year, month) as
+select * from functional.alltypes
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=4.03MB Threads=2
+Per-Host Resource Estimates: Memory=25MB
+F01:PLAN FRAGMENT [HASH(functional.alltypes.year,functional.alltypes.month)] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=1.03MB mem-reservation=0B thread-reservation=1
+| max-parallelism=10 segment-costs=[3764061]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false, PARTITION-KEYS=(functional.alltypes.year,functional.alltypes.month)]
+| partitions=24
+| output exprs: functional.alltypes.id, functional.alltypes.bool_col, functional.alltypes.tinyint_col, functional.alltypes.smallint_col, functional.alltypes.int_col, functional.alltypes.bigint_col, functional.alltypes.float_col, functional.alltypes.double_col, functional.alltypes.date_string_col, functional.alltypes.string_col, functional.alltypes.timestamp_col, functional.alltypes.year, functional.alltypes.month
+| mem-estimate=63.45KB mem-reservation=0B thread-reservation=0 cost=3760262
+|
+01:EXCHANGE [HASH(functional.alltypes.year,functional.alltypes.month)]
+| mem-estimate=993.45KB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=3799
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 24)
+Per-Instance Resources: mem-estimate=23.63MB mem-reservation=4.03MB thread-reservation=1
+max-parallelism=10 segment-costs=[62944]
+02:TUPLE CACHE
+| cache key: e9f9ad583e91e3b8b4c9755365a3090c
+| input scan node ids: 0
+| estimated serialized size: 662.99KB
+| estimated serialized size per node: 66.30KB
+| cumulative processing cost: 17343
+| cache read processing cost: 970
+| cache write processing cost: 1833
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypes, RANDOM]
+ HDFS partitions=24/24 files=24 size=478.45KB
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=7.30K size=478.45KB
+ partitions: 24/24 rows=7.30K
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=310
+ mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+ tuple-ids=0 row-size=89B cardinality=7.30K cost=17343
+ in pipelines: 00(GETNEXT)
+====
+# IMPALA-4167: non-shuffled CTAS into partitioned table has no exchange node before write.
+# Note that plan hint tests for CTAS are minimal by design, as this logic is covered well
+# in insert.test.
+# Note that noclustered hint is added to ensure consistent plans on Impala 2.x and 3.x,
+# because IMPALA-5293 changed clustered to be the default on 3.x.
+create /*+ noshuffle, noclustered */ table t partitioned by (year, month) as
+select * from functional.alltypes
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=4.03MB Threads=2
+Per-Host Resource Estimates: Memory=25MB
+F01:PLAN FRAGMENT [HASH(functional.alltypes.year,functional.alltypes.month)] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=1.03MB mem-reservation=0B thread-reservation=1
+| max-parallelism=10 segment-costs=[3764061]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false, PARTITION-KEYS=(functional.alltypes.year,functional.alltypes.month)]
+| partitions=24
+| output exprs: functional.alltypes.id, functional.alltypes.bool_col, functional.alltypes.tinyint_col, functional.alltypes.smallint_col, functional.alltypes.int_col, functional.alltypes.bigint_col, functional.alltypes.float_col, functional.alltypes.double_col, functional.alltypes.date_string_col, functional.alltypes.string_col, functional.alltypes.timestamp_col, functional.alltypes.year, functional.alltypes.month
+| mem-estimate=63.45KB mem-reservation=0B thread-reservation=0 cost=3760262
+|
+01:EXCHANGE [HASH(functional.alltypes.year,functional.alltypes.month)]
+| mem-estimate=993.45KB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=3799
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 24)
+Per-Instance Resources: mem-estimate=23.63MB mem-reservation=4.03MB thread-reservation=1
+max-parallelism=10 segment-costs=[62944]
+02:TUPLE CACHE
+| cache key: e9f9ad583e91e3b8b4c9755365a3090c
+| input scan node ids: 0
+| estimated serialized size: 662.99KB
+| estimated serialized size per node: 66.30KB
+| cumulative processing cost: 17343
+| cache read processing cost: 970
+| cache write processing cost: 1833
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=89B cardinality=7.30K cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [functional.alltypes, RANDOM]
+ HDFS partitions=24/24 files=24 size=478.45KB
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=7.30K size=478.45KB
+ partitions: 24/24 rows=7.30K
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=310
+ mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+ tuple-ids=0 row-size=89B cardinality=7.30K cost=17343
+ in pipelines: 00(GETNEXT)
+====
+# CTAS with more complex select query
+create table t partitioned by (c_nationkey) sort by (c_custkey) as
+select c_custkey, max(o_totalprice) as maxprice, c_nationkey
+ from tpch.orders join tpch.customer on c_custkey = o_custkey
+ where c_nationkey < 10
+ group by c_custkey, c_nationkey
+---- PLAN
+Max Per-Host Resource Reservation: Memory=34.94MB Threads=1
+Per-Host Resource Estimates: Memory=51MB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=50.94MB mem-reservation=34.94MB thread-reservation=1 runtime-filters-memory=1.00MB
+| max-parallelism=1 segment-costs=[8333477, 178515, 4409218]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false, PARTITION-KEYS=(c_nationkey)]
+| partitions=25
+| output exprs: c_custkey, max(o_totalprice), c_nationkey
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=4409218
+|
+04:SORT
+| order by: c_nationkey ASC NULLS LAST, c_custkey ASC NULLS LAST
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=18B cardinality=150.00K cost=178515
+| in pipelines: 04(GETNEXT), 03(OPEN)
+|
+08:TUPLE CACHE
+| cache key: 5ce44149bb5c81be120f796766ab013d
+| input scan node ids: 0,1
+| estimated serialized size: 3.15MB
+| estimated serialized size per node: 1.57MB
+| cumulative processing cost: 8333477
+| cache read processing cost: 19935
+| cache write processing cost: 8910
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=18B cardinality=150.00K cost=0
+| in pipelines: 03(GETNEXT)
+|
+03:AGGREGATE [FINALIZE]
+| output: max(o_totalprice)
+| group by: c_custkey, c_nationkey
+| mem-estimate=10.00MB mem-reservation=8.50MB spill-buffer=512.00KB thread-reservation=0
+| tuple-ids=2 row-size=18B cardinality=150.00K cost=993506
+| in pipelines: 03(GETNEXT), 00(OPEN)
+|
+07:TUPLE CACHE
+| cache key: 331dbec25b9283e2a5af31415d97bb81
+| input scan node ids: 0,1
+| estimated serialized size: 7.41MB
+| estimated serialized size per node: 3.71MB
+| cumulative processing cost: 7339971
+| cache read processing cost: 30391
+| cache write processing cost: 20993
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0,1 row-size=26B cardinality=228.68K cost=0
+| in pipelines: 00(GETNEXT)
+|
+02:HASH JOIN [INNER JOIN]
+| hash predicates: o_custkey = c_custkey
+| fk/pk conjuncts: o_custkey = c_custkey
+| runtime filters: RF000[bloom] <- c_custkey
+| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=0,1 row-size=26B cardinality=228.68K cost=441187
+| in pipelines: 00(GETNEXT), 01(OPEN)
+|
+|--06:TUPLE CACHE
+| | cache key: 729fff08b06858466430f2b1bbae07b0
+| | input scan node ids: 1
+| | estimated serialized size: 205.08KB
+| | estimated serialized size per node: 205.08KB
+| | cumulative processing cost: 864778
+| | cache read processing cost: 1993
+| | cache write processing cost: 567
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=1 row-size=10B cardinality=15.00K cost=0
+| | in pipelines: 01(GETNEXT)
+| |
+| 01:SCAN HDFS [tpch.customer]
+| HDFS partitions=1/1 files=1 size=23.08MB
+| deterministic scan range assignment: true
+| predicates: c_nationkey < CAST(10 AS SMALLINT)
+| stored statistics:
+| table: rows=150.00K size=23.08MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=150.00K
+| mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+| tuple-ids=1 row-size=10B cardinality=15.00K cost=864778
+| in pipelines: 01(GETNEXT)
+|
+05:TUPLE CACHE
+| cache key: 8310644f189aa574a97e008385767db3
+| input scan node ids: 0
+| estimated serialized size: 28.61MB
+| estimated serialized size per node: 14.31MB
+| cumulative processing cost: 6034006
+| cache read processing cost: 199350
+| cache write processing cost: 81000
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=16B cardinality=1.50M cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [tpch.orders]
+ HDFS partitions=1/1 files=1 size=162.56MB
+ deterministic scan range assignment: true
+ runtime filters: RF000[bloom] -> o_custkey
+ stored statistics:
+ table: rows=1.50M size=162.56MB
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=1.18M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=16B cardinality=1.50M cost=6034006
+ in pipelines: 00(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=51.69MB Threads=5
+Per-Host Resource Estimates: Memory=81MB
+F03:PLAN FRAGMENT [HASH(c_nationkey)] hosts=2 instances=2
+| Per-Instance Resources: mem-estimate=7.33MB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=2 segment-costs=[207600, 4409218] cpu-comparison-result=3 [max(2 (self) vs 3 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.t, OVERWRITE=false, PARTITION-KEYS=(c_nationkey)]
+| partitions=25
+| output exprs: c_custkey, max(o_totalprice), c_nationkey
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=4409218
+|
+08:SORT
+| order by: c_nationkey ASC NULLS LAST, c_custkey ASC NULLS LAST
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=18B cardinality=150.00K cost=178515
+| in pipelines: 08(GETNEXT), 06(OPEN)
+|
+07:EXCHANGE [HASH(c_nationkey)]
+| mem-estimate=1.33MB mem-reservation=0B thread-reservation=0
+| tuple-ids=2 row-size=18B cardinality=150.00K cost=29085
+| in pipelines: 06(GETNEXT)
+|
+F02:PLAN FRAGMENT [HASH(c_custkey,c_nationkey)] hosts=2 instances=2
+Per-Instance Resources: mem-estimate=11.42MB mem-reservation=4.75MB thread-reservation=1
+max-parallelism=2 segment-costs=[930185, 251160] cpu-comparison-result=3 [max(2 (self) vs 3 (sum children))]
+06:AGGREGATE [FINALIZE]
+| output: max:merge(o_totalprice)
+| group by: c_custkey, c_nationkey
+| mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB thread-reservation=0
+| tuple-ids=2 row-size=18B cardinality=150.00K cost=899158
+| in pipelines: 06(GETNEXT), 00(OPEN)
+|
+05:EXCHANGE [HASH(c_custkey,c_nationkey)]
+| mem-estimate=1.42MB mem-reservation=0B thread-reservation=0
+| tuple-ids=2 row-size=18B cardinality=160.02K cost=31027
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
+Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+Per-Instance Resources: mem-estimate=38.17MB mem-reservation=25.00MB thread-reservation=1
+max-parallelism=2 segment-costs=[7499075, 267937] cpu-comparison-result=3 [max(2 (self) vs 3 (sum children))]
+12:TUPLE CACHE
+| cache key: 595303d6a82ff6cbe552f87c2917c40c
+| input scan node ids: 0
+| estimated serialized size: 3.36MB
+| estimated serialized size per node: 1.68MB
+| cumulative processing cost: 8383568
+| cache read processing cost: 21266
+| cache write processing cost: 9505
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=18B cardinality=160.02K cost=0
+| in pipelines: 00(GETNEXT)
+|
+03:AGGREGATE [STREAMING]
+| output: max(o_totalprice)
+| group by: c_custkey, c_nationkey
+| mem-estimate=10.00MB mem-reservation=5.00MB spill-buffer=256.00KB thread-reservation=0
+| tuple-ids=2 row-size=18B cardinality=160.02K cost=1038882
+| in pipelines: 00(GETNEXT)
+|
+11:TUPLE CACHE
+| cache key: 2e5e647f00b10107ed9b14d4295d422c
+| input scan node ids: 0
+| estimated serialized size: 7.41MB
+| estimated serialized size per node: 3.71MB
+| cumulative processing cost: 7344686
+| cache read processing cost: 30391
+| cache write processing cost: 20993
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0,1 row-size=26B cardinality=228.68K cost=0
+| in pipelines: 00(GETNEXT)
+|
+02:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=00
+| hash predicates: o_custkey = c_custkey
+| fk/pk conjuncts: o_custkey = c_custkey
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=0,1 row-size=26B cardinality=228.68K cost=426187
+| in pipelines: 00(GETNEXT), 01(OPEN)
+|
+|--F04:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
+| | Per-Instance Resources: mem-estimate=3.09MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=2 segment-costs=[18986]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: c_custkey
+| | runtime filters: RF000[bloom] <- c_custkey
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0 cost=15000
+| |
+| 04:EXCHANGE [BROADCAST]
+| | mem-estimate=160.48KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=1 row-size=10B cardinality=15.00K cost=3986
+| | in pipelines: 01(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=20.05MB mem-reservation=12.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[865507]
+| 10:TUPLE CACHE
+| | cache key: 729fff08b06858466430f2b1bbae07b0
+| | input scan node ids: 1
+| | estimated serialized size: 205.08KB
+| | estimated serialized size per node: 205.08KB
+| | cumulative processing cost: 864778
+| | cache read processing cost: 1993
+| | cache write processing cost: 567
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=1 row-size=10B cardinality=15.00K cost=0
+| | in pipelines: 01(GETNEXT)
+| |
+| 01:SCAN HDFS [tpch.customer, RANDOM]
+| HDFS partitions=1/1 files=1 size=23.08MB
+| deterministic scan range assignment: true
+| predicates: c_nationkey < CAST(10 AS SMALLINT)
+| stored statistics:
+| table: rows=150.00K size=23.08MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=150.00K
+| mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+| tuple-ids=1 row-size=10B cardinality=15.00K cost=864778
+| in pipelines: 01(GETNEXT)
+|
+09:TUPLE CACHE
+| cache key: 3d784b6bccae4e83dcf0209e1faa1314
+| input scan node ids: 0
+| estimated serialized size: 28.61MB
+| estimated serialized size per node: 14.31MB
+| cumulative processing cost: 6034006
+| cache read processing cost: 199350
+| cache write processing cost: 81000
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0 row-size=16B cardinality=1.50M cost=0
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [tpch.orders, RANDOM]
+ HDFS partitions=1/1 files=1 size=162.56MB
+ deterministic scan range assignment: true
+ runtime filters: RF000[bloom] -> o_custkey
+ stored statistics:
+ table: rows=1.50M size=162.56MB
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=1.18M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=16B cardinality=1.50M cost=6034006
+ in pipelines: 00(GETNEXT)
+====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-ddl-iceberg.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-ddl-iceberg.test
new file mode 100644
index 000000000..9bba63810
--- /dev/null
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-ddl-iceberg.test
@@ -0,0 +1,796 @@
+# Unpartitioned insert.
+create table store_sales_unpartitioned
+stored as iceberg as
+select * from store_sales
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=96.00MB Threads=12
+Per-Host Resource Estimates: Memory=193MB
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=16.10MB mem-reservation=8.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[108991552732]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_unpartitioned, OVERWRITE=false]
+| output exprs: tpcds_partitioned_parquet_snap.store_sales.ss_sold_time_sk, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_hdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_addr_sk, tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, tpcds_partitioned_parquet_snap.store_sales.ss_promo_sk, tpcds_partitioned_parq [...]
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=97047706322
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=8.64G cost=11943846410
+ in pipelines: 00(GETNEXT)
+====
+# Unpartitioned insert with less than 256MB estimated data to write.
+create table store_sales_unpartitioned_small
+stored as iceberg as
+select * from store_sales
+where ss_sold_date_sk = 2450816
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=176.00KB Threads=1
+Per-Host Resource Estimates: Memory=16MB
+F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=16.10MB mem-reservation=176.00KB thread-reservation=1
+| max-parallelism=1 segment-costs=[40845904]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_unpartitioned_small, OVERWRITE=false]
+| output exprs: tpcds_partitioned_parquet_snap.store_sales.ss_sold_time_sk, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_hdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_addr_sk, tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, tpcds_partitioned_parquet_snap.store_sales.ss_promo_sk, tpcds_partitioned_parq [...]
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=36802982
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk = CAST(2450816 AS INT)
+ HDFS partitions=1/1824 files=1 size=218.89MB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1/1 rows=2.92M
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=8.64G
+ mem-estimate=16.00MB mem-reservation=176.00KB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=2.92M cost=4042922
+ in pipelines: 00(GETNEXT)
+====
+# Unpartitioned insert with writer & scan instances < maximum allowed (120).
+# Cost-wise, scanner only need 40 instances, but writer need 88 instances.
+create table store_sales_unpartitioned_medium
+stored as iceberg as
+select * from store_sales
+where ss_sold_date_sk < 2450910
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=1.55MB Threads=9
+Per-Host Resource Estimates: Memory=145MB
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=88
+| Per-Instance Resources: mem-estimate=16.10MB mem-reservation=176.00KB thread-reservation=1
+| max-parallelism=88 segment-costs=[3108264904]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_unpartitioned_medium, OVERWRITE=false]
+| output exprs: tpcds_partitioned_parquet_snap.store_sales.ss_sold_time_sk, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_hdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_addr_sk, tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, tpcds_partitioned_parquet_snap.store_sales.ss_promo_sk, tpcds_partitioned_parq [...]
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=2768066475
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk < CAST(2450910 AS INT)
+ HDFS partitions=93/1824 files=93 size=19.88GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 93/93 rows=246.09M
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=119.03M
+ mem-estimate=16.00MB mem-reservation=176.00KB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=246.09M cost=340198429
+ in pipelines: 00(GETNEXT)
+====
+# Same unpartitioned insert as before, but with low MAX_FS_WRITERS.
+create table store_sales_unpartitioned_medium
+stored as iceberg as
+select * from store_sales
+where ss_sold_date_sk < 2450910
+---- QUERYOPTIONS
+MAX_FS_WRITERS=20
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=1.72MB Threads=12
+Per-Host Resource Estimates: Memory=276MB
+F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=20
+| Per-Instance Resources: mem-estimate=19.18MB mem-reservation=0B thread-reservation=1
+| max-parallelism=20 segment-costs=[2904081862]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_unpartitioned_medium, OVERWRITE=false]
+| output exprs: tpcds_partitioned_parquet_snap.store_sales.ss_sold_time_sk, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_hdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_addr_sk, tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, tpcds_partitioned_parquet_snap.store_sales.ss_promo_sk, tpcds_partitioned_parq [...]
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=2768066475
+|
+01:EXCHANGE [RANDOM]
+| mem-estimate=19.08MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=96B cardinality=246.09M cost=136015387
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=93
+Per-Instance Resources: mem-estimate=23.81MB mem-reservation=176.00KB thread-reservation=1
+max-parallelism=93 segment-costs=[1988428320]
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk < CAST(2450910 AS INT)
+ HDFS partitions=93/1824 files=93 size=19.88GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 93/93 rows=246.09M
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=119.03M
+ mem-estimate=16.00MB mem-reservation=176.00KB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=246.09M cost=340198429
+ in pipelines: 00(GETNEXT)
+====
+# Partition by ss_sold_date_sk, just like the original table.
+create table store_sales_duplicate partitioned by (ss_sold_date_sk)
+stored as iceberg as
+select * from store_sales
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=168.00MB Threads=24
+Per-Host Resource Estimates: Memory=50.99GB
+F01:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=4.19GB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[44372981821, 97047706322]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_duplicate, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=97047706322
+|
+02:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=4.17GB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=96B cardinality=8.64G cost=39597689640
+| in pipelines: 02(GETNEXT), 00(OPEN)
+|
+01:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk)]
+| mem-estimate=21.72MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=96B cardinality=8.64G cost=4775292181
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=62.88MB mem-reservation=8.00MB thread-reservation=1
+max-parallelism=1824 segment-costs=[69810676358]
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=8.64G cost=11943846410
+ in pipelines: 00(GETNEXT)
+====
+# Partition by ss_sold_date_sk, but limit at 5 rows.
+create table store_sales_5_rows partitioned by (ss_sold_date_sk)
+stored as iceberg as
+select * from store_sales
+limit 5
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=14.00MB Threads=2
+Per-Host Resource Estimates: Memory=22MB
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=6.02MB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[24, 3954291]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_5_rows, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk
+| mem-estimate=480B mem-reservation=0B thread-reservation=0 cost=3954291
+|
+02:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=96B cardinality=5 cost=22
+| in pipelines: 02(GETNEXT), 00(OPEN)
+|
+01:EXCHANGE [UNPARTITIONED]
+| limit: 5
+| mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=96B cardinality=5 cost=2
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 120)
+Per-Instance Resources: mem-estimate=16.39MB mem-reservation=8.00MB thread-reservation=1
+max-parallelism=10 segment-costs=[39]
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ limit: 5
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=5 cost=6
+ in pipelines: 00(GETNEXT)
+====
+# Partition by ss_sold_date_sk, but have only 1 row per partition.
+create table store_sales_1_row_per_part partitioned by (ss_sold_date_sk)
+stored as iceberg as
+select count(*), ss_sold_date_sk
+from store_sales group by ss_sold_date_sk
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=14.06MB Threads=2
+Per-Host Resource Estimates: Memory=47MB
+F01:PLAN FRAGMENT [HASH(ss_sold_date_sk)] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=16.00MB mem-reservation=7.94MB thread-reservation=1
+| max-parallelism=10 segment-costs=[5592, 1694, 3956795] cpu-comparison-result=10 [max(10 (self) vs 10 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_1_row_per_part, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| output exprs: count(*), ss_sold_date_sk
+| mem-estimate=2.13KB mem-reservation=0B thread-reservation=0 cost=3956795
+|
+04:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=12B cardinality=1.82K cost=1694
+| in pipelines: 04(GETNEXT), 03(OPEN)
+|
+03:AGGREGATE [FINALIZE]
+| output: count:merge(*)
+| group by: ss_sold_date_sk
+| mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1 row-size=12B cardinality=1.82K cost=5289
+| in pipelines: 03(GETNEXT), 00(OPEN)
+|
+02:EXCHANGE [HASH(ss_sold_date_sk)]
+| mem-estimate=162.14KB mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=12B cardinality=1.82K cost=303
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 120)
+Per-Instance Resources: mem-estimate=30.62MB mem-reservation=6.12MB thread-reservation=1
+max-parallelism=10 segment-costs=[5604, 2349]
+05:TUPLE CACHE
+| cache key: 0f28ed31aa6856c81f1e044cb23ef2d3
+| input scan node ids: 0
+| estimated serialized size: 28.50KB
+| estimated serialized size per node: 2.85KB
+| cumulative processing cost: 5604
+| cache read processing cost: 242
+| cache write processing cost: 78
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=12B cardinality=1.82K cost=0
+| in pipelines: 00(GETNEXT)
+|
+01:AGGREGATE [STREAMING]
+| output: sum_init_zero(tpcds_partitioned_parquet_snap.store_sales.stats: num_rows)
+| group by: ss_sold_date_sk
+| mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1 row-size=12B cardinality=1.82K cost=5289
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+ tuple-ids=0 row-size=12B cardinality=1.82K cost=315
+ in pipelines: 00(GETNEXT)
+====
+# Partition by ss_sold_date_sk, but have insert 5 partition and 1 row per partition.
+create table store_sales_5_part_1_row_per_part partitioned by (ss_sold_date_sk)
+stored as iceberg as
+select count(*), ss_store_sk, ss_sold_date_sk
+from store_sales group by ss_store_sk, ss_sold_date_sk
+limit 5
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=470.62MB Threads=16
+Per-Host Resource Estimates: Memory=701MB
+F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=6.02MB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[5, 3954244] cpu-comparison-result=120 [max(1 (self) vs 120 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_5_part_1_row_per_part, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| output exprs: count(*), ss_store_sk, ss_sold_date_sk
+| mem-estimate=80B mem-reservation=0B thread-reservation=0 cost=3954244
+|
+05:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=16B cardinality=5 cost=5
+| in pipelines: 05(GETNEXT), 03(OPEN)
+|
+04:EXCHANGE [UNPARTITIONED]
+| limit: 5
+| mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=16B cardinality=5 cost=0
+| in pipelines: 03(GETNEXT)
+|
+F01:PLAN FRAGMENT [HASH(ss_store_sk,ss_sold_date_sk)] hosts=10 instances=30 (adjusted from 120)
+Per-Instance Resources: mem-estimate=22.34MB mem-reservation=2.88MB thread-reservation=1
+max-parallelism=30 segment-costs=[228190932, 7] cpu-comparison-result=120 [max(30 (self) vs 120 (sum children))]
+03:AGGREGATE [FINALIZE]
+| output: count:merge(*)
+| group by: ss_store_sk, ss_sold_date_sk
+| limit: 5
+| mem-estimate=10.00MB mem-reservation=2.88MB spill-buffer=128.00KB thread-reservation=0
+| tuple-ids=1 row-size=16B cardinality=5 cost=201792013
+| in pipelines: 03(GETNEXT), 00(OPEN)
+|
+02:EXCHANGE [HASH(ss_store_sk,ss_sold_date_sk)]
+| mem-estimate=12.34MB mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=16B cardinality=142.93M cost=26398919
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=52.34MB mem-reservation=38.00MB thread-reservation=1
+max-parallelism=1360 segment-costs=[13514707828, 220910505]
+01:AGGREGATE [STREAMING]
+| output: count(*)
+| group by: ss_store_sk, ss_sold_date_sk
+| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=16B cardinality=142.93M cost=12519387294
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=4.00MB thread-reservation=0
+ tuple-ids=0 row-size=8B cardinality=8.64G cost=995320534
+ in pipelines: 00(GETNEXT)
+====
+# Number of partition & scan instances < maximum allowed (120).
+create table store_sales_83_part partitioned by (ss_sold_date_sk)
+stored as iceberg as
+select * from store_sales
+where ss_sold_date_sk < 2450900
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=49.55MB Threads=17
+Per-Host Resource Estimates: Memory=2.56GB
+F01:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk)] hosts=10 instances=78
+| Per-Instance Resources: mem-estimate=274.87MB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=78 segment-costs=[1123464765, 2460972835]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_83_part, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=2460972835
+|
+02:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=256.76MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=96B cardinality=218.75M cost=1002560725
+| in pipelines: 02(GETNEXT), 00(OPEN)
+|
+01:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk)]
+| mem-estimate=18.11MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=96B cardinality=218.75M cost=120904040
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=83
+Per-Instance Resources: mem-estimate=46.47MB mem-reservation=176.00KB thread-reservation=1
+max-parallelism=83 segment-costs=[1767513380]
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk < CAST(2450900 AS INT)
+ HDFS partitions=83/1824 files=83 size=17.74GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 83/83 rows=218.75M
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=133.91M
+ mem-estimate=16.00MB mem-reservation=176.00KB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=218.75M cost=302402289
+ in pipelines: 00(GETNEXT)
+====
+# Number of partition & scan instances < num executors.
+create table store_sales_4_part partitioned by (ss_sold_date_sk)
+stored as iceberg as
+select * from store_sales
+where ss_sold_date_sk < 2450820
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=6.17MB Threads=2
+Per-Host Resource Estimates: Memory=287MB
+F01:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk)] hosts=4 instances=4
+| Per-Instance Resources: mem-estimate=269.22MB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=4 segment-costs=[58077317, 130969386]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_4_part, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=130969386
+|
+02:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=258.83MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=96B cardinality=11.31M cost=51827204
+| in pipelines: 02(GETNEXT), 00(OPEN)
+|
+01:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk)]
+| mem-estimate=10.39MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=96B cardinality=11.31M cost=6250113
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+Per-Instance Resources: mem-estimate=17.56MB mem-reservation=176.00KB thread-reservation=1
+max-parallelism=4 segment-costs=[91371298]
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk < CAST(2450820 AS INT)
+ HDFS partitions=4/1824 files=4 size=875.57MB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 4/4 rows=11.31M
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=2.61G
+ mem-estimate=16.00MB mem-reservation=176.00KB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=11.31M cost=15632634
+ in pipelines: 00(GETNEXT)
+====
+# Partition number is 1.
+# There should be no shuffling.
+create table store_sales_1_part partitioned by (ss_sold_date_sk)
+stored as iceberg as
+select * from store_sales
+where ss_sold_date_sk = 2450816
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=6.17MB Threads=1
+Per-Host Resource Estimates: Memory=284MB
+F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=283.75MB mem-reservation=6.17MB thread-reservation=1
+| max-parallelism=1 segment-costs=[17446509, 36802982]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_1_part, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=36802982
+|
+01:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=267.75MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=96B cardinality=2.92M cost=13403587
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk = CAST(2450816 AS INT)
+ HDFS partitions=1/1824 files=1 size=218.89MB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1/1 rows=2.92M
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=8.64G
+ mem-estimate=16.00MB mem-reservation=176.00KB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=2.92M cost=4042922
+ in pipelines: 00(GETNEXT)
+====
+# Partition number is 1 with large number of rows.
+# NDV(ca_country) = 1
+# There should be no shuffling.
+create table customer_address_1_huge_part partitioned by (part_col)
+stored as iceberg as
+select a.*, a.ca_country as part_col
+from customer_address as a, customer_address as b
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=12.25MB Threads=3
+Per-Host Resource Estimates: Memory=50.03GB
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=50.02GB mem-reservation=12.12MB thread-reservation=1
+| max-parallelism=10 segment-costs=[2393532728547225, 2622165786501232] cpu-comparison-result=20 [max(10 (self) vs 20 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.customer_address_1_huge_part, OVERWRITE=false, PARTITION-KEYS=(ca_country)]
+| output exprs: ca_address_sk, ca_address_id, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type, ca_country
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=2622165786501232
+|
+04:SORT
+| order by: ca_country ASC NULLS LAST
+| mem-estimate=50.00GB mem-reservation=12.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=4 row-size=232B cardinality=225.00T cost=2358455207032019
+| in pipelines: 04(GETNEXT), 00(OPEN)
+|
+02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
+| join table id: 00
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=0,1 row-size=232B cardinality=225.00T cost=35077500000000
+| in pipelines: 00(GETNEXT), 01(OPEN)
+|
+|--F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=40.00KB mem-reservation=0B thread-reservation=1
+| | max-parallelism=10 segment-costs=[19934990]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0 cost=0
+| |
+| 03:EXCHANGE [BROADCAST]
+| | mem-estimate=40.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=1 row-size=0B cardinality=15.00M cost=19934990
+| | in pipelines: 01(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=16.02MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=10 segment-costs=[324000]
+| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.customer_address b, RANDOM]
+| HDFS partitions=1/1 files=1 size=307.36MB
+| stored statistics:
+| table: rows=15.00M size=307.36MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=1.58M
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=1 row-size=0B cardinality=15.00M cost=0
+| in pipelines: 01(GETNEXT)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.customer_address a, RANDOM]
+ HDFS partitions=1/1 files=1 size=307.36MB
+ stored statistics:
+ table: rows=15.00M size=307.36MB
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=1.58M
+ mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+ tuple-ids=0 row-size=232B cardinality=15.00M cost=21515206
+ in pipelines: 00(GETNEXT)
+====
+# Partition number is 1 (constant expression) with large number of rows.
+# Since the partitioning col is a constant expression, it is recognized as
+# non-partitioned insert and maximum num writer is scheduled.
+# Along with maximum number of scan scheduled, shuffling in-between is eliminated.
+create table store_sales_1_huge_part partitioned by (part_col)
+stored as iceberg as
+select a.*, 100000 as part_col
+from store_sales a, store_sales b;
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=96.25MB Threads=15
+Per-Host Resource Estimates: Memory=225MB
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=16.10MB mem-reservation=8.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[2517058240805469066] cpu-comparison-result=120 [max(120 (self) vs 30 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_1_huge_part, OVERWRITE=false, PARTITION-KEYS=(100000)]
+| output exprs: a.ss_sold_time_sk, a.ss_item_sk, a.ss_customer_sk, a.ss_cdemo_sk, a.ss_hdemo_sk, a.ss_addr_sk, a.ss_store_sk, a.ss_promo_sk, a.ss_ticket_number, a.ss_quantity, a.ss_wholesale_cost, a.ss_list_price, a.ss_sales_price, a.ss_ext_discount_amt, a.ss_ext_sales_price, a.ss_ext_wholesale_cost, a.ss_ext_list_price, a.ss_ext_tax, a.ss_coupon_amt, a.ss_net_paid, a.ss_net_paid_inc_tax, a.ss_net_profit, a.ss_sold_date_sk, CAST(100000 AS INT)
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=1079134528315963008
+|
+02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
+| join table id: 00
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=0,1 row-size=96B cardinality=9223372.04T cost=1437923700545659648
+| in pipelines: 00(GETNEXT), 01(OPEN)
+|
+|--F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=80.00KB mem-reservation=0B thread-reservation=1
+| | max-parallelism=10 segment-costs=[11482473870]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0 cost=0
+| |
+| 03:EXCHANGE [BROADCAST]
+| | mem-estimate=80.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=1 row-size=0B cardinality=8.64G cost=11482473870
+| | in pipelines: 01(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=20 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=16.02MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=20 segment-costs=[186622600]
+| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales b, RANDOM]
+| HDFS partitions=1824/1824 files=1824 size=389.90GB
+| stored statistics:
+| table: rows=8.64G size=389.90GB
+| partitions: 1824/1824 rows=8.64G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=390.22M
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=1 row-size=0B cardinality=8.64G cost=0
+| in pipelines: 01(GETNEXT)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales a, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=8.64G cost=11943846410
+ in pipelines: 00(GETNEXT)
+====
+# Partition number is 4 with large number of rows.
+# Since the partitioning col is a arithmetic expression, it is recognized as
+# partitioned insert, but the expression is not evaluated further to lower partition
+# estimate down from 1824 to 4.
+# Shuffling in-between is kept because data partitioning between F02 and F00 is different.
+create table store_sales_4_huge_part partitioned by (part_col)
+stored as iceberg as
+select a.*, (a.ss_sold_date_sk % 4) as part_col
+from store_sales a, store_sales b;
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=168.25MB Threads=27
+Per-Host Resource Estimates: Memory=51.05GB
+F02:PLAN FRAGMENT [HASH((a.ss_sold_date_sk % 4))] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=4.19GB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[9223372036854775807, 1079134528315963008] cpu-comparison-result=240 [max(240 (self) vs 30 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_4_huge_part, OVERWRITE=false, PARTITION-KEYS=((ss_sold_date_sk % 4))]
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk, (ss_sold_date_sk % CAST(4 AS INT))
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=1079134528315963008
+|
+05:SORT
+| order by: (ss_sold_date_sk % 4) ASC NULLS LAST
+| mem-estimate=4.17GB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=4 row-size=96B cardinality=9223372.04T cost=9223372036854775807
+| in pipelines: 05(GETNEXT), 00(OPEN)
+|
+04:EXCHANGE [HASH((a.ss_sold_date_sk % 4))]
+| mem-estimate=22.19MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0,1 row-size=96B cardinality=9223372.04T cost=727724053707841792
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=64.75MB mem-reservation=8.00MB thread-reservation=1
+max-parallelism=1824 segment-costs=[2031908871662953610]
+02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
+| join table id: 00
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=0,1 row-size=96B cardinality=9223372.04T cost=1437923700545659648
+| in pipelines: 00(GETNEXT), 01(OPEN)
+|
+|--F03:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=80.00KB mem-reservation=0B thread-reservation=1
+| | max-parallelism=10 segment-costs=[11482473870]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0 cost=0
+| |
+| 03:EXCHANGE [BROADCAST]
+| | mem-estimate=80.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=1 row-size=0B cardinality=8.64G cost=11482473870
+| | in pipelines: 01(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=20 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=16.02MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=20 segment-costs=[186622600]
+| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales b, RANDOM]
+| HDFS partitions=1824/1824 files=1824 size=389.90GB
+| stored statistics:
+| table: rows=8.64G size=389.90GB
+| partitions: 1824/1824 rows=8.64G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=390.22M
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=1 row-size=0B cardinality=8.64G cost=0
+| in pipelines: 01(GETNEXT)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales a, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=8.64G cost=11943846410
+ in pipelines: 00(GETNEXT)
+====
+# Partition number is 20 with large number of rows.
+# NDV(cs_ship_mode_sk) = 20.
+# Shuffling in-between is kept because data partitioning between F02 and F00 is different.
+create table catalog_sales_20_huge_part partitioned by (part_col)
+stored as iceberg as
+select a.*, a.cs_ship_mode_sk as part_col
+from catalog_sales a, catalog_sales b;
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=24.12MB Threads=16
+Per-Host Resource Estimates: Memory=50.39GB
+F02:PLAN FRAGMENT [HASH(a.cs_ship_mode_sk)] hosts=10 instances=20
+| Per-Instance Resources: mem-estimate=25.03GB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=20 segment-costs=[9223372036854775807, 1079134528315963008] cpu-comparison-result=140 [max(140 (self) vs 20 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.catalog_sales_20_huge_part, OVERWRITE=false, PARTITION-KEYS=(cs_ship_mode_sk)]
+| output exprs: cs_sold_time_sk, cs_ship_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_bill_addr_sk, cs_ship_customer_sk, cs_ship_cdemo_sk, cs_ship_hdemo_sk, cs_ship_addr_sk, cs_call_center_sk, cs_catalog_page_sk, cs_ship_mode_sk, cs_warehouse_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_list_price, cs_sales_price, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price, cs_ext_tax, cs_coupon_amt, cs_ex [...]
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=1079134528315963008
+|
+05:SORT
+| order by: cs_ship_mode_sk ASC NULLS LAST
+| mem-estimate=25.00GB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=4 row-size=140B cardinality=9223372.04T cost=9223372036854775807
+| in pipelines: 05(GETNEXT), 00(OPEN)
+|
+04:EXCHANGE [HASH(a.cs_ship_mode_sk)]
+| mem-estimate=27.34MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0,1 row-size=140B cardinality=9223372.04T cost=727724053707841792
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=27.56MB mem-reservation=1.00MB thread-reservation=1
+max-parallelism=1831 segment-costs=[2031908868428384405]
+02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
+| join table id: 00
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=0,1 row-size=140B cardinality=9223372.04T cost=1437923700545659648
+| in pipelines: 00(GETNEXT), 01(OPEN)
+|
+|--F03:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=40.00KB mem-reservation=0B thread-reservation=1
+| | max-parallelism=10 segment-costs=[5741383630]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0 cost=0
+| |
+| 03:EXCHANGE [BROADCAST]
+| | mem-estimate=40.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=1 row-size=0B cardinality=4.32G cost=5741383630
+| | in pipelines: 01(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=16.02MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=10 segment-costs=[93313684]
+| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales b, RANDOM]
+| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| stored statistics:
+| table: rows=4.32G size=280.96GB
+| partitions: 1831/1831 rows=4.32G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=21.52M
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=1 row-size=0B cardinality=4.32G cost=0
+| in pipelines: 01(GETNEXT)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales a, RANDOM]
+ HDFS partitions=1831/1831 files=1831 size=280.96GB
+ stored statistics:
+ table: rows=4.32G size=280.96GB
+ partitions: 1831/1831 rows=4.32G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=21.52M
+ mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+ tuple-ids=0 row-size=140B cardinality=4.32G cost=8709277205
+ in pipelines: 00(GETNEXT)
+====
+# Partition with cardinality = 0.
+create table store_sales_zero_cardinality partitioned by (ss_sold_date_sk)
+stored as iceberg as
+select * from store_sales
+where ss_sold_date_sk = 1
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=6.00MB Threads=1
+Per-Host Resource Estimates: Memory=10MB
+F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=6.00MB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[0, 3954235]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_zero_cardinality, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk
+| mem-estimate=96B mem-reservation=0B thread-reservation=0 cost=3954235
+|
+01:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=96B cardinality=0 cost=0
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk = CAST(1 AS INT)
+ partitions=0/1824 files=0 size=0B
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 0/0 rows=unavailable
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=0
+ mem-estimate=0B mem-reservation=0B thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=0 cost=0
+ in pipelines: 00(GETNEXT)
+====
+# Partitioned insert with unavailable stats.
+# tpcds_seq_snap does not have stats collected.
+# Exchange node must exist in query plan.
+create table store_sales_without_stats partitioned by (part_col)
+stored as iceberg as
+select *, ss_item_sk as part_col
+from tpcds_seq_snap.store_sales
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=102.00MB Threads=13
+Per-Host Resource Estimates: Memory=391MB
+F01:PLAN FRAGMENT [HASH(ss_item_sk)] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=150.19MB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=10 segment-costs=[0, 3954235]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_without_stats, OVERWRITE=false, PARTITION-KEYS=(ss_item_sk)]
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk, ss_item_sk
+| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 cost=3954235
+|
+02:SORT
+| order by: ss_item_sk ASC NULLS LAST
+| mem-estimate=128.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=100B cardinality=unavailable cost=0
+| in pipelines: 02(GETNEXT), 00(OPEN)
+|
+01:EXCHANGE [HASH(ss_item_sk)]
+| mem-estimate=22.19MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=100B cardinality=unavailable cost=0
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=20.06MB mem-reservation=8.00MB thread-reservation=1
+max-parallelism=120 segment-costs=[1200000000]
+00:SCAN HDFS [tpcds_seq_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=215.99MB
+ stored statistics:
+ table: rows=unavailable size=unavailable
+ partitions: 0/1824 rows=unavailable
+ columns missing stats: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit
+ extrapolated-rows=disabled max-scan-range-rows=unavailable
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=100B cardinality=unavailable cost=1200000000
+ in pipelines: 00(GETNEXT)
+====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-ddl-parquet.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-ddl-parquet.test
new file mode 100644
index 000000000..610240505
--- /dev/null
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-ddl-parquet.test
@@ -0,0 +1,813 @@
+# Unpartitioned insert.
+create table store_sales_unpartitioned
+stored as parquet as
+select * from store_sales
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=96.00MB Threads=12
+Per-Host Resource Estimates: Memory=12.19GB
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.02GB mem-reservation=8.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[108991552732]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_unpartitioned, OVERWRITE=false]
+| partitions=1
+| output exprs: tpcds_partitioned_parquet_snap.store_sales.ss_sold_time_sk, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_hdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_addr_sk, tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, tpcds_partitioned_parquet_snap.store_sales.ss_promo_sk, tpcds_partitioned_parq [...]
+| mem-estimate=1.00GB mem-reservation=0B thread-reservation=0 cost=97047706322
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=8.64G cost=11943846410
+ in pipelines: 00(GETNEXT)
+====
+# Unpartitioned insert with less than 256MB estimated data to write.
+create table store_sales_unpartitioned_small
+stored as parquet as
+select * from store_sales
+where ss_sold_date_sk = 2450816
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=176.00KB Threads=1
+Per-Host Resource Estimates: Memory=284MB
+F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=283.75MB mem-reservation=176.00KB thread-reservation=1
+| max-parallelism=1 segment-costs=[40845904]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_unpartitioned_small, OVERWRITE=false]
+| partitions=1
+| output exprs: tpcds_partitioned_parquet_snap.store_sales.ss_sold_time_sk, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_hdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_addr_sk, tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, tpcds_partitioned_parquet_snap.store_sales.ss_promo_sk, tpcds_partitioned_parq [...]
+| mem-estimate=267.75MB mem-reservation=0B thread-reservation=0 cost=36802982
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk = CAST(2450816 AS INT)
+ HDFS partitions=1/1824 files=1 size=218.89MB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1/1 rows=2.92M
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=8.64G
+ mem-estimate=16.00MB mem-reservation=176.00KB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=2.92M cost=4042922
+ in pipelines: 00(GETNEXT)
+====
+# Unpartitioned insert with writer & scan instances < maximum allowed (120).
+# Cost-wise, scanner only need 40 instances, but writer need 88 instances.
+create table store_sales_unpartitioned_medium
+stored as parquet as
+select * from store_sales
+where ss_sold_date_sk < 2450910
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=1.55MB Threads=9
+Per-Host Resource Estimates: Memory=2.39GB
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=88
+| Per-Instance Resources: mem-estimate=272.03MB mem-reservation=176.00KB thread-reservation=1
+| max-parallelism=88 segment-costs=[3108264904]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_unpartitioned_medium, OVERWRITE=false]
+| partitions=1
+| output exprs: tpcds_partitioned_parquet_snap.store_sales.ss_sold_time_sk, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_hdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_addr_sk, tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, tpcds_partitioned_parquet_snap.store_sales.ss_promo_sk, tpcds_partitioned_parq [...]
+| mem-estimate=256.03MB mem-reservation=0B thread-reservation=0 cost=2768066475
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk < CAST(2450910 AS INT)
+ HDFS partitions=93/1824 files=93 size=19.88GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 93/93 rows=246.09M
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=119.03M
+ mem-estimate=16.00MB mem-reservation=176.00KB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=246.09M cost=340198429
+ in pipelines: 00(GETNEXT)
+====
+# Same unpartitioned insert as before, but with low MAX_FS_WRITERS.
+create table store_sales_unpartitioned_medium
+stored as parquet as
+select * from store_sales
+where ss_sold_date_sk < 2450910
+---- QUERYOPTIONS
+MAX_FS_WRITERS=20
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=1.72MB Threads=12
+Per-Host Resource Estimates: Memory=2.27GB
+F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=20
+| Per-Instance Resources: mem-estimate=1.02GB mem-reservation=0B thread-reservation=1
+| max-parallelism=20 segment-costs=[2904081862]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_unpartitioned_medium, OVERWRITE=false]
+| partitions=1
+| output exprs: tpcds_partitioned_parquet_snap.store_sales.ss_sold_time_sk, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_hdemo_sk, tpcds_partitioned_parquet_snap.store_sales.ss_addr_sk, tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, tpcds_partitioned_parquet_snap.store_sales.ss_promo_sk, tpcds_partitioned_parq [...]
+| mem-estimate=1.00GB mem-reservation=0B thread-reservation=0 cost=2768066475
+|
+01:EXCHANGE [RANDOM]
+| mem-estimate=19.08MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=96B cardinality=246.09M cost=136015387
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=93
+Per-Instance Resources: mem-estimate=23.81MB mem-reservation=176.00KB thread-reservation=1
+max-parallelism=93 segment-costs=[1988428320]
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk < CAST(2450910 AS INT)
+ HDFS partitions=93/1824 files=93 size=19.88GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 93/93 rows=246.09M
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=119.03M
+ mem-estimate=16.00MB mem-reservation=176.00KB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=246.09M cost=340198429
+ in pipelines: 00(GETNEXT)
+====
+# Partition by ss_sold_date_sk, just like the original table.
+create table store_sales_duplicate partitioned by (ss_sold_date_sk)
+stored as parquet as
+select * from store_sales
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=168.00MB Threads=24
+Per-Host Resource Estimates: Memory=62.74GB
+F01:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=5.17GB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[44372981821, 97047706322]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_duplicate, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| partitions=1824
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk
+| mem-estimate=1.00GB mem-reservation=0B thread-reservation=0 cost=97047706322
+|
+02:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=4.17GB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=96B cardinality=8.64G cost=39597689640
+| in pipelines: 02(GETNEXT), 00(OPEN)
+|
+01:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk)]
+| mem-estimate=21.72MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=96B cardinality=8.64G cost=4775292181
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=62.88MB mem-reservation=8.00MB thread-reservation=1
+max-parallelism=1824 segment-costs=[69810676358]
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=8.64G cost=11943846410
+ in pipelines: 00(GETNEXT)
+====
+# Partition by ss_sold_date_sk, but limit at 5 rows.
+create table store_sales_5_rows partitioned by (ss_sold_date_sk)
+stored as parquet as
+select * from store_sales
+limit 5
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=14.00MB Threads=2
+Per-Host Resource Estimates: Memory=22MB
+F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=6.02MB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[24, 3954291]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_5_rows, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| partitions=1824
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk
+| mem-estimate=480B mem-reservation=0B thread-reservation=0 cost=3954291
+|
+02:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=96B cardinality=5 cost=22
+| in pipelines: 02(GETNEXT), 00(OPEN)
+|
+01:EXCHANGE [UNPARTITIONED]
+| limit: 5
+| mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=96B cardinality=5 cost=2
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 120)
+Per-Instance Resources: mem-estimate=16.39MB mem-reservation=8.00MB thread-reservation=1
+max-parallelism=10 segment-costs=[39]
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ limit: 5
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=5 cost=6
+ in pipelines: 00(GETNEXT)
+====
+# Partition by ss_sold_date_sk, but have only 1 row per partition.
+create table store_sales_1_row_per_part partitioned by (ss_sold_date_sk)
+stored as parquet as
+select count(*), ss_sold_date_sk
+from store_sales group by ss_sold_date_sk
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=14.06MB Threads=2
+Per-Host Resource Estimates: Memory=47MB
+F01:PLAN FRAGMENT [HASH(ss_sold_date_sk)] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=16.00MB mem-reservation=7.94MB thread-reservation=1
+| max-parallelism=10 segment-costs=[5592, 1694, 3956795] cpu-comparison-result=10 [max(10 (self) vs 10 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_1_row_per_part, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| partitions=1824
+| output exprs: count(*), ss_sold_date_sk
+| mem-estimate=2.13KB mem-reservation=0B thread-reservation=0 cost=3956795
+|
+04:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=12B cardinality=1.82K cost=1694
+| in pipelines: 04(GETNEXT), 03(OPEN)
+|
+03:AGGREGATE [FINALIZE]
+| output: count:merge(*)
+| group by: ss_sold_date_sk
+| mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1 row-size=12B cardinality=1.82K cost=5289
+| in pipelines: 03(GETNEXT), 00(OPEN)
+|
+02:EXCHANGE [HASH(ss_sold_date_sk)]
+| mem-estimate=162.14KB mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=12B cardinality=1.82K cost=303
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 120)
+Per-Instance Resources: mem-estimate=30.62MB mem-reservation=6.12MB thread-reservation=1
+max-parallelism=10 segment-costs=[5604, 2349]
+05:TUPLE CACHE
+| cache key: 0f28ed31aa6856c81f1e044cb23ef2d3
+| input scan node ids: 0
+| estimated serialized size: 28.50KB
+| estimated serialized size per node: 2.85KB
+| cumulative processing cost: 5604
+| cache read processing cost: 242
+| cache write processing cost: 78
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=12B cardinality=1.82K cost=0
+| in pipelines: 00(GETNEXT)
+|
+01:AGGREGATE [STREAMING]
+| output: sum_init_zero(tpcds_partitioned_parquet_snap.store_sales.stats: num_rows)
+| group by: ss_sold_date_sk
+| mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1 row-size=12B cardinality=1.82K cost=5289
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ deterministic scan range assignment: true
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+ tuple-ids=0 row-size=12B cardinality=1.82K cost=315
+ in pipelines: 00(GETNEXT)
+====
+# Partition by ss_sold_date_sk, but have insert 5 partition and 1 row per partition.
+create table store_sales_5_part_1_row_per_part partitioned by (ss_sold_date_sk)
+stored as parquet as
+select count(*), ss_store_sk, ss_sold_date_sk
+from store_sales group by ss_store_sk, ss_sold_date_sk
+limit 5
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=470.62MB Threads=16
+Per-Host Resource Estimates: Memory=701MB
+F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=6.02MB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[5, 3954244] cpu-comparison-result=120 [max(1 (self) vs 120 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_5_part_1_row_per_part, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| partitions=1824
+| output exprs: count(*), ss_store_sk, ss_sold_date_sk
+| mem-estimate=80B mem-reservation=0B thread-reservation=0 cost=3954244
+|
+05:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=16B cardinality=5 cost=5
+| in pipelines: 05(GETNEXT), 03(OPEN)
+|
+04:EXCHANGE [UNPARTITIONED]
+| limit: 5
+| mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=16B cardinality=5 cost=0
+| in pipelines: 03(GETNEXT)
+|
+F01:PLAN FRAGMENT [HASH(ss_store_sk,ss_sold_date_sk)] hosts=10 instances=30 (adjusted from 120)
+Per-Instance Resources: mem-estimate=22.34MB mem-reservation=2.88MB thread-reservation=1
+max-parallelism=30 segment-costs=[228190932, 7] cpu-comparison-result=120 [max(30 (self) vs 120 (sum children))]
+03:AGGREGATE [FINALIZE]
+| output: count:merge(*)
+| group by: ss_store_sk, ss_sold_date_sk
+| limit: 5
+| mem-estimate=10.00MB mem-reservation=2.88MB spill-buffer=128.00KB thread-reservation=0
+| tuple-ids=1 row-size=16B cardinality=5 cost=201792013
+| in pipelines: 03(GETNEXT), 00(OPEN)
+|
+02:EXCHANGE [HASH(ss_store_sk,ss_sold_date_sk)]
+| mem-estimate=12.34MB mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=16B cardinality=142.93M cost=26398919
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=52.34MB mem-reservation=38.00MB thread-reservation=1
+max-parallelism=1360 segment-costs=[13514707828, 220910505]
+01:AGGREGATE [STREAMING]
+| output: count(*)
+| group by: ss_store_sk, ss_sold_date_sk
+| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=16B cardinality=142.93M cost=12519387294
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=4.00MB thread-reservation=0
+ tuple-ids=0 row-size=8B cardinality=8.64G cost=995320534
+ in pipelines: 00(GETNEXT)
+====
+# Number of partition & scan instances < maximum allowed (120).
+create table store_sales_83_part partitioned by (ss_sold_date_sk)
+stored as parquet as
+select * from store_sales
+where ss_sold_date_sk < 2450900
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=49.55MB Threads=17
+Per-Host Resource Estimates: Memory=4.42GB
+F01:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk)] hosts=10 instances=78
+| Per-Instance Resources: mem-estimate=513.52MB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=78 segment-costs=[1123464765, 2460972835]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_83_part, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| partitions=1824
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk
+| mem-estimate=256.76MB mem-reservation=0B thread-reservation=0 cost=2460972835
+|
+02:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=256.76MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=96B cardinality=218.75M cost=1002560725
+| in pipelines: 02(GETNEXT), 00(OPEN)
+|
+01:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk)]
+| mem-estimate=18.11MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=96B cardinality=218.75M cost=120904040
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=83
+Per-Instance Resources: mem-estimate=46.47MB mem-reservation=176.00KB thread-reservation=1
+max-parallelism=83 segment-costs=[1767513380]
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk < CAST(2450900 AS INT)
+ HDFS partitions=83/1824 files=83 size=17.74GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 83/83 rows=218.75M
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=133.91M
+ mem-estimate=16.00MB mem-reservation=176.00KB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=218.75M cost=302402289
+ in pipelines: 00(GETNEXT)
+====
+# Number of partition & scan instances < num executors.
+create table store_sales_4_part partitioned by (ss_sold_date_sk)
+stored as parquet as
+select * from store_sales
+where ss_sold_date_sk < 2450820
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=6.17MB Threads=2
+Per-Host Resource Estimates: Memory=535MB
+F01:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk)] hosts=4 instances=4
+| Per-Instance Resources: mem-estimate=517.65MB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=4 segment-costs=[58077317, 130969386]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_4_part, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| partitions=1824
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk
+| mem-estimate=258.83MB mem-reservation=0B thread-reservation=0 cost=130969386
+|
+02:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=258.83MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=96B cardinality=11.31M cost=51827204
+| in pipelines: 02(GETNEXT), 00(OPEN)
+|
+01:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk)]
+| mem-estimate=10.39MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=96B cardinality=11.31M cost=6250113
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+Per-Instance Resources: mem-estimate=17.56MB mem-reservation=176.00KB thread-reservation=1
+max-parallelism=4 segment-costs=[91371298]
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk < CAST(2450820 AS INT)
+ HDFS partitions=4/1824 files=4 size=875.57MB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 4/4 rows=11.31M
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=2.61G
+ mem-estimate=16.00MB mem-reservation=176.00KB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=11.31M cost=15632634
+ in pipelines: 00(GETNEXT)
+====
+# Partition number is 1.
+# There should be no shuffling.
+create table store_sales_1_part partitioned by (ss_sold_date_sk)
+stored as parquet as
+select * from store_sales
+where ss_sold_date_sk = 2450816
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=6.17MB Threads=1
+Per-Host Resource Estimates: Memory=536MB
+F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=535.50MB mem-reservation=6.17MB thread-reservation=1
+| max-parallelism=1 segment-costs=[17446509, 36802982]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_1_part, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| partitions=1824
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk
+| mem-estimate=267.75MB mem-reservation=0B thread-reservation=0 cost=36802982
+|
+01:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=267.75MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=2 row-size=96B cardinality=2.92M cost=13403587
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk = CAST(2450816 AS INT)
+ HDFS partitions=1/1824 files=1 size=218.89MB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1/1 rows=2.92M
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=8.64G
+ mem-estimate=16.00MB mem-reservation=176.00KB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=2.92M cost=4042922
+ in pipelines: 00(GETNEXT)
+====
+# Partition number is 1 with large number of rows.
+# NDV(ca_country) = 1
+# There should be no shuffling.
+create table customer_address_1_huge_part partitioned by (part_col)
+stored as parquet as
+select a.*, a.ca_country as part_col
+from customer_address as a, customer_address as b
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=12.25MB Threads=3
+Per-Host Resource Estimates: Memory=51.02GB
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=51.00GB mem-reservation=12.12MB thread-reservation=1
+| max-parallelism=10 segment-costs=[2393532728547225, 2622165786501232] cpu-comparison-result=20 [max(10 (self) vs 20 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.customer_address_1_huge_part, OVERWRITE=false, PARTITION-KEYS=(ca_country)]
+| partitions=1
+| output exprs: ca_address_sk, ca_address_id, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type, ca_country
+| mem-estimate=1.00GB mem-reservation=0B thread-reservation=0 cost=2622165786501232
+|
+04:SORT
+| order by: ca_country ASC NULLS LAST
+| mem-estimate=50.00GB mem-reservation=12.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=4 row-size=232B cardinality=225.00T cost=2358455207032019
+| in pipelines: 04(GETNEXT), 00(OPEN)
+|
+02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
+| join table id: 00
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=0,1 row-size=232B cardinality=225.00T cost=35077500000000
+| in pipelines: 00(GETNEXT), 01(OPEN)
+|
+|--F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=40.00KB mem-reservation=0B thread-reservation=1
+| | max-parallelism=10 segment-costs=[19934990]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0 cost=0
+| |
+| 03:EXCHANGE [BROADCAST]
+| | mem-estimate=40.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=1 row-size=0B cardinality=15.00M cost=19934990
+| | in pipelines: 01(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=16.02MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=10 segment-costs=[324000]
+| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.customer_address b, RANDOM]
+| HDFS partitions=1/1 files=1 size=307.36MB
+| stored statistics:
+| table: rows=15.00M size=307.36MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=1.58M
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=1 row-size=0B cardinality=15.00M cost=0
+| in pipelines: 01(GETNEXT)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.customer_address a, RANDOM]
+ HDFS partitions=1/1 files=1 size=307.36MB
+ stored statistics:
+ table: rows=15.00M size=307.36MB
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=1.58M
+ mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+ tuple-ids=0 row-size=232B cardinality=15.00M cost=21515206
+ in pipelines: 00(GETNEXT)
+====
+# Partition number is 1 (constant expression) with large number of rows.
+# Since the partitioning col is a constant expression, it is recognized as
+# non-partitioned insert and maximum num writer is scheduled.
+# Along with maximum number of scan scheduled, shuffling in-between is eliminated.
+create table store_sales_1_huge_part partitioned by (part_col)
+stored as parquet as
+select a.*, 100000 as part_col
+from store_sales a, store_sales b;
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=96.25MB Threads=15
+Per-Host Resource Estimates: Memory=12.22GB
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.02GB mem-reservation=8.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[2517058240805469066] cpu-comparison-result=120 [max(120 (self) vs 30 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_1_huge_part, OVERWRITE=false, PARTITION-KEYS=(100000)]
+| partitions=1
+| output exprs: a.ss_sold_time_sk, a.ss_item_sk, a.ss_customer_sk, a.ss_cdemo_sk, a.ss_hdemo_sk, a.ss_addr_sk, a.ss_store_sk, a.ss_promo_sk, a.ss_ticket_number, a.ss_quantity, a.ss_wholesale_cost, a.ss_list_price, a.ss_sales_price, a.ss_ext_discount_amt, a.ss_ext_sales_price, a.ss_ext_wholesale_cost, a.ss_ext_list_price, a.ss_ext_tax, a.ss_coupon_amt, a.ss_net_paid, a.ss_net_paid_inc_tax, a.ss_net_profit, a.ss_sold_date_sk, CAST(100000 AS INT)
+| mem-estimate=1.00GB mem-reservation=0B thread-reservation=0 cost=1079134528315963008
+|
+02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
+| join table id: 00
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=0,1 row-size=96B cardinality=9223372.04T cost=1437923700545659648
+| in pipelines: 00(GETNEXT), 01(OPEN)
+|
+|--F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=80.00KB mem-reservation=0B thread-reservation=1
+| | max-parallelism=10 segment-costs=[11482473870]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0 cost=0
+| |
+| 03:EXCHANGE [BROADCAST]
+| | mem-estimate=80.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=1 row-size=0B cardinality=8.64G cost=11482473870
+| | in pipelines: 01(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=20 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=16.02MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=20 segment-costs=[186622600]
+| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales b, RANDOM]
+| HDFS partitions=1824/1824 files=1824 size=389.90GB
+| stored statistics:
+| table: rows=8.64G size=389.90GB
+| partitions: 1824/1824 rows=8.64G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=390.22M
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=1 row-size=0B cardinality=8.64G cost=0
+| in pipelines: 01(GETNEXT)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales a, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=8.64G cost=11943846410
+ in pipelines: 00(GETNEXT)
+====
+# Partition number is 4 with large number of rows.
+# Since the partitioning col is a arithmetic expression, it is recognized as
+# partitioned insert, but the expression is not evaluated further to lower partition
+# estimate down from 1824 to 4.
+# Shuffling in-between is kept because data partitioning between F02 and F00 is different.
+create table store_sales_4_huge_part partitioned by (part_col)
+stored as parquet as
+select a.*, (a.ss_sold_date_sk % 4) as part_col
+from store_sales a, store_sales b;
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=168.25MB Threads=27
+Per-Host Resource Estimates: Memory=62.79GB
+F02:PLAN FRAGMENT [HASH((a.ss_sold_date_sk % 4))] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=5.17GB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[9223372036854775807, 1079134528315963008] cpu-comparison-result=240 [max(240 (self) vs 30 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_4_huge_part, OVERWRITE=false, PARTITION-KEYS=((ss_sold_date_sk % 4))]
+| partitions=1824
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk, (ss_sold_date_sk % CAST(4 AS INT))
+| mem-estimate=1.00GB mem-reservation=0B thread-reservation=0 cost=1079134528315963008
+|
+05:SORT
+| order by: (ss_sold_date_sk % 4) ASC NULLS LAST
+| mem-estimate=4.17GB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=4 row-size=96B cardinality=9223372.04T cost=9223372036854775807
+| in pipelines: 05(GETNEXT), 00(OPEN)
+|
+04:EXCHANGE [HASH((a.ss_sold_date_sk % 4))]
+| mem-estimate=22.19MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0,1 row-size=96B cardinality=9223372.04T cost=727724053707841792
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=64.75MB mem-reservation=8.00MB thread-reservation=1
+max-parallelism=1824 segment-costs=[2031908871662953610]
+02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
+| join table id: 00
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=0,1 row-size=96B cardinality=9223372.04T cost=1437923700545659648
+| in pipelines: 00(GETNEXT), 01(OPEN)
+|
+|--F03:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=80.00KB mem-reservation=0B thread-reservation=1
+| | max-parallelism=10 segment-costs=[11482473870]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0 cost=0
+| |
+| 03:EXCHANGE [BROADCAST]
+| | mem-estimate=80.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=1 row-size=0B cardinality=8.64G cost=11482473870
+| | in pipelines: 01(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=20 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=16.02MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=20 segment-costs=[186622600]
+| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales b, RANDOM]
+| HDFS partitions=1824/1824 files=1824 size=389.90GB
+| stored statistics:
+| table: rows=8.64G size=389.90GB
+| partitions: 1824/1824 rows=8.64G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=390.22M
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=1 row-size=0B cardinality=8.64G cost=0
+| in pipelines: 01(GETNEXT)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales a, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=8.64G cost=11943846410
+ in pipelines: 00(GETNEXT)
+====
+# Partition number is 20 with large number of rows.
+# NDV(cs_ship_mode_sk) = 20.
+# Shuffling in-between is kept because data partitioning between F02 and F00 is different.
+create table catalog_sales_20_huge_part partitioned by (part_col)
+stored as parquet as
+select a.*, a.cs_ship_mode_sk as part_col
+from catalog_sales a, catalog_sales b;
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=24.12MB Threads=16
+Per-Host Resource Estimates: Memory=52.34GB
+F02:PLAN FRAGMENT [HASH(a.cs_ship_mode_sk)] hosts=10 instances=20
+| Per-Instance Resources: mem-estimate=26.00GB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=20 segment-costs=[9223372036854775807, 1079134528315963008] cpu-comparison-result=140 [max(140 (self) vs 20 (sum children))]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.catalog_sales_20_huge_part, OVERWRITE=false, PARTITION-KEYS=(cs_ship_mode_sk)]
+| partitions=20
+| output exprs: cs_sold_time_sk, cs_ship_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_bill_addr_sk, cs_ship_customer_sk, cs_ship_cdemo_sk, cs_ship_hdemo_sk, cs_ship_addr_sk, cs_call_center_sk, cs_catalog_page_sk, cs_ship_mode_sk, cs_warehouse_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_list_price, cs_sales_price, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price, cs_ext_tax, cs_coupon_amt, cs_ex [...]
+| mem-estimate=1.00GB mem-reservation=0B thread-reservation=0 cost=1079134528315963008
+|
+05:SORT
+| order by: cs_ship_mode_sk ASC NULLS LAST
+| mem-estimate=25.00GB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=4 row-size=140B cardinality=9223372.04T cost=9223372036854775807
+| in pipelines: 05(GETNEXT), 00(OPEN)
+|
+04:EXCHANGE [HASH(a.cs_ship_mode_sk)]
+| mem-estimate=27.34MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0,1 row-size=140B cardinality=9223372.04T cost=727724053707841792
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=27.56MB mem-reservation=1.00MB thread-reservation=1
+max-parallelism=1831 segment-costs=[2031908868428384405]
+02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
+| join table id: 00
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=0,1 row-size=140B cardinality=9223372.04T cost=1437923700545659648
+| in pipelines: 00(GETNEXT), 01(OPEN)
+|
+|--F03:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=40.00KB mem-reservation=0B thread-reservation=1
+| | max-parallelism=10 segment-costs=[5741383630]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0 cost=0
+| |
+| 03:EXCHANGE [BROADCAST]
+| | mem-estimate=40.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=1 row-size=0B cardinality=4.32G cost=5741383630
+| | in pipelines: 01(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=10 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=16.02MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=10 segment-costs=[93313684]
+| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales b, RANDOM]
+| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| stored statistics:
+| table: rows=4.32G size=280.96GB
+| partitions: 1831/1831 rows=4.32G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=21.52M
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=1 row-size=0B cardinality=4.32G cost=0
+| in pipelines: 01(GETNEXT)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales a, RANDOM]
+ HDFS partitions=1831/1831 files=1831 size=280.96GB
+ stored statistics:
+ table: rows=4.32G size=280.96GB
+ partitions: 1831/1831 rows=4.32G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=21.52M
+ mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+ tuple-ids=0 row-size=140B cardinality=4.32G cost=8709277205
+ in pipelines: 00(GETNEXT)
+====
+# Partition with cardinality = 0.
+create table store_sales_zero_cardinality partitioned by (ss_sold_date_sk)
+stored as parquet as
+select * from store_sales
+where ss_sold_date_sk = 1
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=6.00MB Threads=1
+Per-Host Resource Estimates: Memory=10MB
+F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=6.00MB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[0, 3954235]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_zero_cardinality, OVERWRITE=false, PARTITION-KEYS=(ss_sold_date_sk)]
+| partitions=1824
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk
+| mem-estimate=96B mem-reservation=0B thread-reservation=0 cost=3954235
+|
+01:SORT
+| order by: ss_sold_date_sk ASC NULLS LAST
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=96B cardinality=0 cost=0
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ partition predicates: ss_sold_date_sk = CAST(1 AS INT)
+ partitions=0/1824 files=0 size=0B
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 0/0 rows=unavailable
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=0
+ mem-estimate=0B mem-reservation=0B thread-reservation=0
+ tuple-ids=0 row-size=96B cardinality=0 cost=0
+ in pipelines: 00(GETNEXT)
+====
+# Partitioned insert with unavailable stats.
+# tpcds_seq_snap does not have stats collected.
+# Exchange node must exist in query plan.
+create table store_sales_without_stats partitioned by (part_col)
+stored as parquet as
+select *, ss_item_sk as part_col
+from tpcds_seq_snap.store_sales
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=102.00MB Threads=13
+Per-Host Resource Estimates: Memory=1.36GB
+F01:PLAN FRAGMENT [HASH(ss_item_sk)] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=1.12GB mem-reservation=6.00MB thread-reservation=1
+| max-parallelism=10 segment-costs=[0, 3954235]
+WRITE TO HDFS [tpcds_partitioned_parquet_snap.store_sales_without_stats, OVERWRITE=false, PARTITION-KEYS=(ss_item_sk)]
+| partitions=unavailable
+| output exprs: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk, ss_item_sk
+| mem-estimate=1.00GB mem-reservation=0B thread-reservation=0 cost=3954235
+|
+02:SORT
+| order by: ss_item_sk ASC NULLS LAST
+| mem-estimate=128.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=100B cardinality=unavailable cost=0
+| in pipelines: 02(GETNEXT), 00(OPEN)
+|
+01:EXCHANGE [HASH(ss_item_sk)]
+| mem-estimate=22.19MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0 row-size=100B cardinality=unavailable cost=0
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=20.06MB mem-reservation=8.00MB thread-reservation=1
+max-parallelism=120 segment-costs=[1200000000]
+00:SCAN HDFS [tpcds_seq_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=215.99MB
+ stored statistics:
+ table: rows=unavailable size=unavailable
+ partitions: 0/1824 rows=unavailable
+ columns missing stats: ss_sold_time_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_list_price, ss_sales_price, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price, ss_ext_tax, ss_coupon_amt, ss_net_paid, ss_net_paid_inc_tax, ss_net_profit
+ extrapolated-rows=disabled max-scan-range-rows=unavailable
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=0 row-size=100B cardinality=unavailable cost=1200000000
+ in pipelines: 00(GETNEXT)
+====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q01.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q01.test
new file mode 100644
index 000000000..011a4cc65
--- /dev/null
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q01.test
@@ -0,0 +1,902 @@
+# TPCDS-Q01
+# start query 1 in stream 0 using template query1.tpl using seed 2031708268
+with customer_total_return as
+(select sr_customer_sk as ctr_customer_sk
+,sr_store_sk as ctr_store_sk
+,sum(SR_FEE) as ctr_total_return
+from store_returns
+,date_dim
+where sr_returned_date_sk = d_date_sk
+and d_year =2000
+group by sr_customer_sk
+,sr_store_sk)
+ select c_customer_id
+from customer_total_return ctr1
+,store
+,customer
+where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2
+from customer_total_return ctr2
+where ctr1.ctr_store_sk = ctr2.ctr_store_sk)
+and s_store_sk = ctr1.ctr_store_sk
+and s_state = 'TN'
+and ctr1.ctr_customer_sk = c_customer_sk
+order by c_customer_id
+limit 100;
+
+# end query 1 in stream 0 using template query1.tpl
+---- PLAN
+Max Per-Host Resource Reservation: Memory=127.38MB Threads=1
+Per-Host Resource Estimates: Memory=7.16GB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=7.16GB mem-reservation=127.38MB thread-reservation=1 runtime-filters-memory=12.00MB
+| max-parallelism=1 segment-costs=[1056659929, 1056659929, 47039287, 205425661, 100]
+PLAN-ROOT SINK
+| output exprs: c_customer_id
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=100
+|
+14:TOP-N [LIMIT=100]
+| order by: c_customer_id ASC
+| mem-estimate=2.73KB mem-reservation=0B thread-reservation=0
+| tuple-ids=13 row-size=28B cardinality=100 cost=133335611
+| in pipelines: 14(GETNEXT), 05(OPEN)
+|
+19:TUPLE CACHE
+| cache key: 3464f97232593cfa9f9632e1fa804f92
+| input scan node ids: 5,0,1,4,6,7
+| estimated serialized size: 833.30MB
+| estimated serialized size per node: 83.33MB
+| cumulative processing cost: 2232449195
+| cache read processing cost: 1350298
+| cache write processing cost: 2359211
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=5,2,4 row-size=74B cardinality=10.16M cost=0
+| in pipelines: 05(GETNEXT)
+|
+13:HASH JOIN [LEFT SEMI JOIN]
+| hash predicates: sr_store_sk = ctr2.ctr_store_sk
+| other join predicates: sum(SR_FEE) > avg(ctr_total_return) * CAST(1.2 AS DECIMAL(2,1))
+| runtime filters: RF000[bloom] <- ctr2.ctr_store_sk, RF001[min_max] <- ctr2.ctr_store_sk
+| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=5,2,4 row-size=74B cardinality=10.16M cost=4447797
+| in pipelines: 05(GETNEXT), 10(OPEN)
+|
+|--18:TUPLE CACHE
+| | cache key: d975df7358a9b9263f516754e811b766
+| | input scan node ids: 6,7
+| | estimated serialized size: 15.30KB
+| | estimated serialized size per node: 1.53KB
+| | cumulative processing cost: 1103699216
+| | cache read processing cost: 86
+| | cache write processing cost: 42
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=11 row-size=20B cardinality=653 cost=0
+| | in pipelines: 10(GETNEXT)
+| |
+| 10:AGGREGATE [FINALIZE]
+| | output: avg(sum(SR_FEE))
+| | group by: sr_store_sk
+| | mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=11 row-size=20B cardinality=653 cost=47039287
+| | in pipelines: 10(GETNEXT), 09(OPEN)
+| |
+| 09:AGGREGATE [FINALIZE]
+| | output: sum(SR_FEE)
+| | group by: sr_customer_sk, sr_store_sk
+| | mem-estimate=3.24GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=8 row-size=24B cardinality=160.81M cost=949210422
+| | in pipelines: 09(GETNEXT), 06(OPEN)
+| |
+| 08:HASH JOIN [INNER JOIN]
+| | hash predicates: sr_returned_date_sk = d_date_sk
+| | fk/pk conjuncts: sr_returned_date_sk = d_date_sk
+| | runtime filters: RF008[bloom] <- d_date_sk
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=6,7 row-size=24B cardinality=160.81M cost=70387897
+| | in pipelines: 06(GETNEXT), 07(OPEN)
+| |
+| |--17:TUPLE CACHE
+| | | cache key: 5536b7721c3902915addf658472e0caa
+| | | input scan node ids: 7
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=7 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 07(GETNEXT)
+| | |
+| | 07:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=7 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 07(GETNEXT)
+| |
+| 06:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns]
+| HDFS partitions=2004/2004 files=2004 size=48.14GB
+| deterministic scan range assignment: true
+| runtime filters: RF008[bloom] -> sr_returned_date_sk
+| stored statistics:
+| table: rows=863.99M size=48.14GB
+| partitions: 2004/2004 rows=863.99M
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=30.09M est-scan-range=373(filtered from 2004)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=6 row-size=16B cardinality=160.81M(filtered from 863.99M) cost=37051143
+| in pipelines: 06(GETNEXT)
+|
+12:HASH JOIN [INNER JOIN]
+| hash predicates: c_customer_sk = sr_customer_sk
+| fk/pk conjuncts: c_customer_sk = sr_customer_sk
+| runtime filters: RF002[bloom] <- sr_customer_sk, RF003[min_max] <- sr_customer_sk
+| mem-estimate=635.33MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=5,2,4 row-size=74B cardinality=10.16M cost=15748162
+| in pipelines: 05(GETNEXT), 03(OPEN)
+|
+|--11:HASH JOIN [INNER JOIN]
+| | hash predicates: sr_store_sk = s_store_sk
+| | fk/pk conjuncts: sr_store_sk = s_store_sk
+| | runtime filters: RF004[bloom] <- s_store_sk, RF005[min_max] <- s_store_sk
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=2,4 row-size=42B cardinality=11.07M cost=43253937
+| | in pipelines: 03(GETNEXT), 04(OPEN)
+| |
+| |--16:TUPLE CACHE
+| | | cache key: 9eb7b9bfb2906530a439edd9747618b1
+| | | input scan node ids: 4
+| | | estimated serialized size: 1012B
+| | | estimated serialized size per node: 1012B
+| | | cumulative processing cost: 154
+| | | cache read processing cost: 6
+| | | cache write processing cost: 2
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=4 row-size=18B cardinality=46 cost=0
+| | | in pipelines: 04(GETNEXT)
+| | |
+| | 04:SCAN HDFS [tpcds_partitioned_parquet_snap.store]
+| | HDFS partitions=1/1 files=1 size=119.76KB
+| | deterministic scan range assignment: true
+| | predicates: s_state = 'TN'
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store.s_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store.s_store_sk
+| | stored statistics:
+| | table: rows=1.35K size=119.76KB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=1.35K
+| | parquet statistics predicates: s_state = 'TN'
+| | parquet dictionary predicates: s_state = 'TN'
+| | mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
+| | tuple-ids=4 row-size=18B cardinality=46 cost=154
+| | in pipelines: 04(GETNEXT)
+| |
+| 03:AGGREGATE [FINALIZE]
+| | output: sum(SR_FEE)
+| | group by: sr_customer_sk, sr_store_sk
+| | mem-estimate=3.24GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=2 row-size=24B cardinality=160.81M cost=949210422
+| | in pipelines: 03(GETNEXT), 00(OPEN)
+| |
+| 02:HASH JOIN [INNER JOIN]
+| | hash predicates: sr_returned_date_sk = d_date_sk
+| | fk/pk conjuncts: sr_returned_date_sk = d_date_sk
+| | runtime filters: RF006[bloom] <- d_date_sk
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=0,1 row-size=24B cardinality=160.81M cost=70387897
+| | in pipelines: 00(GETNEXT), 01(OPEN)
+| |
+| |--15:TUPLE CACHE
+| | | cache key: 5536b7721c3902915addf658472e0caa
+| | | input scan node ids: 1
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=1 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 01(GETNEXT)
+| | |
+| | 01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=1 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 01(GETNEXT)
+| |
+| 00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns]
+| HDFS partitions=2004/2004 files=2004 size=48.14GB
+| deterministic scan range assignment: true
+| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF005[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF006[bloom] -> sr_returned_date_sk
+| stored statistics:
+| table: rows=863.99M size=48.14GB
+| partitions: 2004/2004 rows=863.99M
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=30.09M est-scan-range=373(filtered from 2004)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=0 row-size=16B cardinality=160.81M(filtered from 863.99M) cost=37051143
+| in pipelines: 00(GETNEXT)
+|
+05:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
+ HDFS partitions=1/1 files=1 size=1.55GB
+ deterministic scan range assignment: true
+ runtime filters: RF003[min_max] -> c_customer_sk, RF002[bloom] -> c_customer_sk
+ stored statistics:
+ table: rows=30.00M size=1.55GB
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=3.10M
+ mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+ tuple-ids=5 row-size=32B cardinality=11.07M(filtered from 30.00M) cost=8640000
+ in pipelines: 05(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=1.60GB Threads=56
+Per-Host Resource Estimates: Memory=3.12GB
+F10:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=4.06MB mem-reservation=4.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[130] cpu-comparison-result=231 [max(1 (self) vs 231 (sum children))]
+PLAN-ROOT SINK
+| output exprs: c_customer_id
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=100
+|
+27:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: c_customer_id ASC
+| limit: 100
+| mem-estimate=62.77KB mem-reservation=0B thread-reservation=0
+| tuple-ids=13 row-size=28B cardinality=100 cost=30
+| in pipelines: 14(GETNEXT)
+|
+F05:PLAN FRAGMENT [HASH(c_customer_sk)] hosts=10 instances=20 (adjusted from 10)
+Per-Instance Resources: mem-estimate=10.35MB mem-reservation=0B thread-reservation=1
+max-parallelism=20 segment-costs=[145321673, 104] cpu-comparison-result=231 [max(30 (self) vs 231 (sum children))]
+14:TOP-N [LIMIT=100]
+| order by: c_customer_id ASC
+| mem-estimate=2.73KB mem-reservation=0B thread-reservation=0
+| tuple-ids=13 row-size=28B cardinality=100 cost=133335611
+| in pipelines: 14(GETNEXT), 05(OPEN)
+|
+13:HASH JOIN [LEFT SEMI JOIN, BROADCAST]
+| hash-table-id=00
+| hash predicates: sr_store_sk = ctr2.ctr_store_sk
+| other join predicates: sum(SR_FEE) > avg(ctr_total_return) * CAST(1.2 AS DECIMAL(2,1))
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=5,2,4 row-size=74B cardinality=10.16M cost=4447144
+| in pipelines: 05(GETNEXT), 25(OPEN)
+|
+|--F11:PLAN FRAGMENT [HASH(c_customer_sk)] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=5.04MB mem-reservation=4.88MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[1513] cpu-comparison-result=110 [max(10 (self) vs 110 (sum children))]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: ctr2.ctr_store_sk
+| | runtime filters: RF000[bloom] <- ctr2.ctr_store_sk, RF001[min_max] <- ctr2.ctr_store_sk
+| | mem-estimate=3.88MB mem-reservation=3.88MB spill-buffer=64.00KB thread-reservation=0 cost=653
+| |
+| 26:EXCHANGE [BROADCAST]
+| | mem-estimate=165.80KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=11 row-size=20B cardinality=653 cost=860
+| | in pipelines: 25(GETNEXT)
+| |
+| F09:PLAN FRAGMENT [HASH(ctr2.ctr_store_sk)] hosts=10 instances=10 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=12.49MB mem-reservation=1.94MB thread-reservation=1
+| max-parallelism=10 segment-costs=[40536, 49] cpu-comparison-result=110 [max(10 (self) vs 110 (sum children))]
+| 25:AGGREGATE [FINALIZE]
+| | output: avg:merge(ctr_total_return)
+| | group by: ctr2.ctr_store_sk
+| | mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=11 row-size=20B cardinality=653 cost=24622
+| | in pipelines: 25(GETNEXT), 23(OPEN)
+| |
+| 24:EXCHANGE [HASH(ctr2.ctr_store_sk)]
+| | mem-estimate=2.49MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=10 row-size=20B cardinality=78.36K cost=15914
+| | in pipelines: 23(GETNEXT)
+| |
+| F08:PLAN FRAGMENT [HASH(sr_customer_sk,sr_store_sk)] hosts=10 instances=100 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=78.22MB mem-reservation=36.00MB thread-reservation=1
+| max-parallelism=100 segment-costs=[984830336, 47241885, 141298] cpu-comparison-result=110 [max(100 (self) vs 110 (sum children))]
+| 10:AGGREGATE [STREAMING]
+| | output: avg(sum(SR_FEE))
+| | group by: sr_store_sk
+| | mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=10 row-size=20B cardinality=78.36K cost=47241885
+| | in pipelines: 23(GETNEXT)
+| |
+| 23:AGGREGATE [FINALIZE]
+| | output: sum:merge(SR_FEE)
+| | group by: sr_customer_sk, sr_store_sk
+| | mem-estimate=55.21MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=8 row-size=24B cardinality=160.81M cost=949210422
+| | in pipelines: 23(GETNEXT), 06(OPEN)
+| |
+| 22:EXCHANGE [HASH(sr_customer_sk,sr_store_sk)]
+| | mem-estimate=13.01MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=8 row-size=24B cardinality=160.81M cost=35619914
+| | in pipelines: 06(GETNEXT)
+| |
+| F06:PLAN FRAGMENT [RANDOM] hosts=10 instances=110 (adjusted from 120)
+| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| Per-Instance Resources: mem-estimate=64.58MB mem-reservation=34.50MB thread-reservation=1
+| max-parallelism=110 segment-costs=[1056649089, 331401897] cpu-comparison-result=110 [max(110 (self) vs 11 (sum children))]
+| 09:AGGREGATE [STREAMING]
+| | output: sum(SR_FEE)
+| | group by: sr_customer_sk, sr_store_sk
+| | mem-estimate=37.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=8 row-size=24B cardinality=160.81M cost=949210422
+| | in pipelines: 06(GETNEXT)
+| |
+| 08:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=01
+| | hash predicates: sr_returned_date_sk = d_date_sk
+| | fk/pk conjuncts: sr_returned_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=6,7 row-size=24B cardinality=160.81M cost=70387524
+| | in pipelines: 06(GETNEXT), 07(OPEN)
+| |
+| |--F12:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=22.33MB mem-reservation=22.31MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=01 plan-id=02 cohort-id=02
+| | | build expressions: d_date_sk
+| | | runtime filters: RF008[bloom] <- d_date_sk
+| | | mem-estimate=21.31MB mem-reservation=21.31MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 21:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=7 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 07(GETNEXT)
+| | |
+| | F07:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 29:TUPLE CACHE
+| | | cache key: 5536b7721c3902915addf658472e0caa
+| | | input scan node ids: 7
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=7 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 07(GETNEXT)
+| | |
+| | 07:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=7 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 07(GETNEXT)
+| |
+| 06:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns, RANDOM]
+| HDFS partitions=2004/2004 files=2004 size=48.14GB
+| runtime filters: RF008[bloom] -> sr_returned_date_sk
+| stored statistics:
+| table: rows=863.99M size=48.14GB
+| partitions: 2004/2004 rows=863.99M
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=30.09M est-scan-range=373(filtered from 2004)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=6 row-size=16B cardinality=160.81M(filtered from 863.99M) cost=37051143
+| in pipelines: 06(GETNEXT)
+|
+12:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=02
+| hash predicates: c_customer_sk = sr_customer_sk
+| fk/pk conjuncts: c_customer_sk = sr_customer_sk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=5,2,4 row-size=74B cardinality=10.16M cost=4680017
+| in pipelines: 05(GETNEXT), 17(OPEN)
+|
+|--F13:PLAN FRAGMENT [HASH(c_customer_sk)] hosts=10 instances=20 (adjusted from 10)
+| | Per-Instance Resources: mem-estimate=56.88MB mem-reservation=42.00MB thread-reservation=1 runtime-filters-memory=8.00MB
+| | max-parallelism=20 segment-costs=[14843489] cpu-comparison-result=121 [max(20 (self) vs 121 (sum children))]
+| JOIN BUILD
+| | join-table-id=02 plan-id=03 cohort-id=01
+| | build expressions: sr_customer_sk
+| | runtime filters: RF002[bloom] <- sr_customer_sk, RF003[min_max] <- sr_customer_sk
+| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=11068145
+| |
+| 20:EXCHANGE [HASH(sr_customer_sk)]
+| | mem-estimate=14.88MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=2,4 row-size=42B cardinality=11.07M cost=3775344
+| | in pipelines: 17(GETNEXT)
+| |
+| F03:PLAN FRAGMENT [HASH(sr_customer_sk,sr_store_sk)] hosts=10 instances=100 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=68.22MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=100 segment-costs=[984830336, 84595626] cpu-comparison-result=121 [max(100 (self) vs 121 (sum children))]
+| 11:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=03
+| | hash predicates: sr_store_sk = s_store_sk
+| | fk/pk conjuncts: sr_store_sk = s_store_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=2,4 row-size=42B cardinality=11.07M cost=43253891
+| | in pipelines: 17(GETNEXT), 04(OPEN)
+| |
+| |--F14:PLAN FRAGMENT [HASH(sr_customer_sk,sr_store_sk)] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=20.39MB mem-reservation=20.38MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[106]
+| | JOIN BUILD
+| | | join-table-id=03 plan-id=04 cohort-id=03
+| | | build expressions: s_store_sk
+| | | runtime filters: RF004[bloom] <- s_store_sk, RF005[min_max] <- s_store_sk
+| | | mem-estimate=19.38MB mem-reservation=19.38MB spill-buffer=64.00KB thread-reservation=0 cost=46
+| | |
+| | 18:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=4 row-size=18B cardinality=46 cost=60
+| | | in pipelines: 04(GETNEXT)
+| | |
+| | F04:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| | Per-Instance Resources: mem-estimate=16.09MB mem-reservation=16.00KB thread-reservation=1
+| | max-parallelism=1 segment-costs=[157]
+| | 04:SCAN HDFS [tpcds_partitioned_parquet_snap.store, RANDOM]
+| | HDFS partitions=1/1 files=1 size=119.76KB
+| | predicates: s_state = 'TN'
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store.s_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store.s_store_sk
+| | stored statistics:
+| | table: rows=1.35K size=119.76KB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=1.35K
+| | parquet statistics predicates: s_state = 'TN'
+| | parquet dictionary predicates: s_state = 'TN'
+| | mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
+| | tuple-ids=4 row-size=18B cardinality=46 cost=154
+| | in pipelines: 04(GETNEXT)
+| |
+| 17:AGGREGATE [FINALIZE]
+| | output: sum:merge(SR_FEE)
+| | group by: sr_customer_sk, sr_store_sk
+| | mem-estimate=55.21MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=2 row-size=24B cardinality=160.81M cost=949210422
+| | in pipelines: 17(GETNEXT), 00(OPEN)
+| |
+| 16:EXCHANGE [HASH(sr_customer_sk,sr_store_sk)]
+| | mem-estimate=13.01MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=2 row-size=24B cardinality=160.81M cost=35619914
+| | in pipelines: 00(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=110 (adjusted from 120)
+| Per-Host Shared Resources: mem-estimate=3.00MB mem-reservation=3.00MB thread-reservation=0 runtime-filters-memory=3.00MB
+| Per-Instance Resources: mem-estimate=64.58MB mem-reservation=34.50MB thread-reservation=1
+| max-parallelism=110 segment-costs=[1056649089, 331401897] cpu-comparison-result=110 [max(110 (self) vs 11 (sum children))]
+| 03:AGGREGATE [STREAMING]
+| | output: sum(SR_FEE)
+| | group by: sr_customer_sk, sr_store_sk
+| | mem-estimate=37.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=2 row-size=24B cardinality=160.81M cost=949210422
+| | in pipelines: 00(GETNEXT)
+| |
+| 02:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=04
+| | hash predicates: sr_returned_date_sk = d_date_sk
+| | fk/pk conjuncts: sr_returned_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=0,1 row-size=24B cardinality=160.81M cost=70387524
+| | in pipelines: 00(GETNEXT), 01(OPEN)
+| |
+| |--F15:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=22.33MB mem-reservation=22.31MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=04 plan-id=05 cohort-id=03
+| | | build expressions: d_date_sk
+| | | runtime filters: RF006[bloom] <- d_date_sk
+| | | mem-estimate=21.31MB mem-reservation=21.31MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 15:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=1 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 01(GETNEXT)
+| | |
+| | F02:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 28:TUPLE CACHE
+| | | cache key: 5536b7721c3902915addf658472e0caa
+| | | input scan node ids: 1
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=1 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 01(GETNEXT)
+| | |
+| | 01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=1 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 01(GETNEXT)
+| |
+| 00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns, RANDOM]
+| HDFS partitions=2004/2004 files=2004 size=48.14GB
+| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF005[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF006[bloom] -> sr_returned_date_sk
+| stored statistics:
+| table: rows=863.99M size=48.14GB
+| partitions: 2004/2004 rows=863.99M
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=30.09M est-scan-range=373(filtered from 2004)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=0 row-size=16B cardinality=160.81M(filtered from 863.99M) cost=37051143
+| in pipelines: 00(GETNEXT)
+|
+19:EXCHANGE [HASH(c_customer_sk)]
+| mem-estimate=10.35MB mem-reservation=0B thread-reservation=0
+| tuple-ids=5 row-size=32B cardinality=11.07M(filtered from 30.00M) cost=2858901
+| in pipelines: 05(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+Per-Host Shared Resources: mem-estimate=8.00MB mem-reservation=8.00MB thread-reservation=0 runtime-filters-memory=8.00MB
+Per-Instance Resources: mem-estimate=18.81MB mem-reservation=1.00MB thread-reservation=1
+max-parallelism=10 segment-costs=[37151541]
+05:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+ HDFS partitions=1/1 files=1 size=1.55GB
+ runtime filters: RF003[min_max] -> c_customer_sk, RF002[bloom] -> c_customer_sk
+ stored statistics:
+ table: rows=30.00M size=1.55GB
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=3.10M
+ mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+ tuple-ids=5 row-size=32B cardinality=11.07M(filtered from 30.00M) cost=8640000
+ in pipelines: 05(GETNEXT)
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=1.60GB Threads=56
+Per-Host Resource Estimates: Memory=3.12GB
+F10:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=4.06MB mem-reservation=4.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[130] cpu-comparison-result=231 [max(1 (self) vs 231 (sum children))]
+PLAN-ROOT SINK
+| output exprs: c_customer_id
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=100
+|
+27:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: c_customer_id ASC
+| limit: 100
+| mem-estimate=62.77KB mem-reservation=0B thread-reservation=0
+| tuple-ids=13 row-size=28B cardinality=100 cost=30
+| in pipelines: 14(GETNEXT)
+|
+F05:PLAN FRAGMENT [HASH(c_customer_sk)] hosts=10 instances=20 (adjusted from 10)
+Per-Instance Resources: mem-estimate=10.35MB mem-reservation=0B thread-reservation=1
+max-parallelism=20 segment-costs=[145321673, 104] cpu-comparison-result=231 [max(30 (self) vs 231 (sum children))]
+14:TOP-N [LIMIT=100]
+| order by: c_customer_id ASC
+| mem-estimate=2.73KB mem-reservation=0B thread-reservation=0
+| tuple-ids=13 row-size=28B cardinality=100 cost=133335611
+| in pipelines: 14(GETNEXT), 05(OPEN)
+|
+13:HASH JOIN [LEFT SEMI JOIN, BROADCAST]
+| hash-table-id=00
+| hash predicates: sr_store_sk = ctr2.ctr_store_sk
+| other join predicates: sum(SR_FEE) > avg(ctr_total_return) * CAST(1.2 AS DECIMAL(2,1))
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=5,2,4 row-size=74B cardinality=10.16M cost=4447144
+| in pipelines: 05(GETNEXT), 25(OPEN)
+|
+|--F11:PLAN FRAGMENT [HASH(c_customer_sk)] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=5.04MB mem-reservation=4.88MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[1513] cpu-comparison-result=110 [max(10 (self) vs 110 (sum children))]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: ctr2.ctr_store_sk
+| | runtime filters: RF000[bloom] <- ctr2.ctr_store_sk, RF001[min_max] <- ctr2.ctr_store_sk
+| | mem-estimate=3.88MB mem-reservation=3.88MB spill-buffer=64.00KB thread-reservation=0 cost=653
+| |
+| 26:EXCHANGE [BROADCAST]
+| | mem-estimate=165.80KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=11 row-size=20B cardinality=653 cost=860
+| | in pipelines: 25(GETNEXT)
+| |
+| F09:PLAN FRAGMENT [HASH(ctr2.ctr_store_sk)] hosts=10 instances=10 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=12.49MB mem-reservation=1.94MB thread-reservation=1
+| max-parallelism=10 segment-costs=[40536, 49] cpu-comparison-result=110 [max(10 (self) vs 110 (sum children))]
+| 25:AGGREGATE [FINALIZE]
+| | output: avg:merge(ctr_total_return)
+| | group by: ctr2.ctr_store_sk
+| | mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=11 row-size=20B cardinality=653 cost=24622
+| | in pipelines: 25(GETNEXT), 23(OPEN)
+| |
+| 24:EXCHANGE [HASH(ctr2.ctr_store_sk)]
+| | mem-estimate=2.49MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=10 row-size=20B cardinality=78.36K cost=15914
+| | in pipelines: 23(GETNEXT)
+| |
+| F08:PLAN FRAGMENT [HASH(sr_customer_sk,sr_store_sk)] hosts=10 instances=100 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=78.22MB mem-reservation=36.00MB thread-reservation=1
+| max-parallelism=100 segment-costs=[984830336, 47241885, 141298] cpu-comparison-result=110 [max(100 (self) vs 110 (sum children))]
+| 10:AGGREGATE [STREAMING]
+| | output: avg(sum(SR_FEE))
+| | group by: sr_store_sk
+| | mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=10 row-size=20B cardinality=78.36K cost=47241885
+| | in pipelines: 23(GETNEXT)
+| |
+| 23:AGGREGATE [FINALIZE]
+| | output: sum:merge(SR_FEE)
+| | group by: sr_customer_sk, sr_store_sk
+| | mem-estimate=55.21MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=8 row-size=24B cardinality=160.81M cost=949210422
+| | in pipelines: 23(GETNEXT), 06(OPEN)
+| |
+| 22:EXCHANGE [HASH(sr_customer_sk,sr_store_sk)]
+| | mem-estimate=13.01MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=8 row-size=24B cardinality=160.81M cost=35619914
+| | in pipelines: 06(GETNEXT)
+| |
+| F06:PLAN FRAGMENT [RANDOM] hosts=10 instances=110 (adjusted from 120)
+| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| Per-Instance Resources: mem-estimate=64.58MB mem-reservation=34.50MB thread-reservation=1
+| max-parallelism=110 segment-costs=[1056649089, 331401897] cpu-comparison-result=110 [max(110 (self) vs 11 (sum children))]
+| 09:AGGREGATE [STREAMING]
+| | output: sum(SR_FEE)
+| | group by: sr_customer_sk, sr_store_sk
+| | mem-estimate=37.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=8 row-size=24B cardinality=160.81M cost=949210422
+| | in pipelines: 06(GETNEXT)
+| |
+| 08:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=01
+| | hash predicates: sr_returned_date_sk = d_date_sk
+| | fk/pk conjuncts: sr_returned_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=6,7 row-size=24B cardinality=160.81M cost=70387524
+| | in pipelines: 06(GETNEXT), 07(OPEN)
+| |
+| |--F12:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=22.33MB mem-reservation=22.31MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=01 plan-id=02 cohort-id=02
+| | | build expressions: d_date_sk
+| | | runtime filters: RF008[bloom] <- d_date_sk
+| | | mem-estimate=21.31MB mem-reservation=21.31MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 21:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=7 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 07(GETNEXT)
+| | |
+| | F07:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 29:TUPLE CACHE
+| | | cache key: 5536b7721c3902915addf658472e0caa
+| | | input scan node ids: 7
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=7 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 07(GETNEXT)
+| | |
+| | 07:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=7 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 07(GETNEXT)
+| |
+| 06:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns, RANDOM]
+| HDFS partitions=2004/2004 files=2004 size=48.14GB
+| runtime filters: RF008[bloom] -> sr_returned_date_sk
+| stored statistics:
+| table: rows=863.99M size=48.14GB
+| partitions: 2004/2004 rows=863.99M
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=30.09M est-scan-range=373(filtered from 2004)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=6 row-size=16B cardinality=160.81M(filtered from 863.99M) cost=37051143
+| in pipelines: 06(GETNEXT)
+|
+12:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=02
+| hash predicates: c_customer_sk = sr_customer_sk
+| fk/pk conjuncts: c_customer_sk = sr_customer_sk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=5,2,4 row-size=74B cardinality=10.16M cost=4680017
+| in pipelines: 05(GETNEXT), 17(OPEN)
+|
+|--F13:PLAN FRAGMENT [HASH(c_customer_sk)] hosts=10 instances=20 (adjusted from 10)
+| | Per-Instance Resources: mem-estimate=56.88MB mem-reservation=42.00MB thread-reservation=1 runtime-filters-memory=8.00MB
+| | max-parallelism=20 segment-costs=[14843489] cpu-comparison-result=121 [max(20 (self) vs 121 (sum children))]
+| JOIN BUILD
+| | join-table-id=02 plan-id=03 cohort-id=01
+| | build expressions: sr_customer_sk
+| | runtime filters: RF002[bloom] <- sr_customer_sk, RF003[min_max] <- sr_customer_sk
+| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=11068145
+| |
+| 20:EXCHANGE [HASH(sr_customer_sk)]
+| | mem-estimate=14.88MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=2,4 row-size=42B cardinality=11.07M cost=3775344
+| | in pipelines: 17(GETNEXT)
+| |
+| F03:PLAN FRAGMENT [HASH(sr_customer_sk,sr_store_sk)] hosts=10 instances=100 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=68.22MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=100 segment-costs=[984830336, 84595626] cpu-comparison-result=121 [max(100 (self) vs 121 (sum children))]
+| 11:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=03
+| | hash predicates: sr_store_sk = s_store_sk
+| | fk/pk conjuncts: sr_store_sk = s_store_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=2,4 row-size=42B cardinality=11.07M cost=43253891
+| | in pipelines: 17(GETNEXT), 04(OPEN)
+| |
+| |--F14:PLAN FRAGMENT [HASH(sr_customer_sk,sr_store_sk)] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=20.39MB mem-reservation=20.38MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[106]
+| | JOIN BUILD
+| | | join-table-id=03 plan-id=04 cohort-id=03
+| | | build expressions: s_store_sk
+| | | runtime filters: RF004[bloom] <- s_store_sk, RF005[min_max] <- s_store_sk
+| | | mem-estimate=19.38MB mem-reservation=19.38MB spill-buffer=64.00KB thread-reservation=0 cost=46
+| | |
+| | 18:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=4 row-size=18B cardinality=46 cost=60
+| | | in pipelines: 04(GETNEXT)
+| | |
+| | F04:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| | Per-Instance Resources: mem-estimate=16.09MB mem-reservation=16.00KB thread-reservation=1
+| | max-parallelism=1 segment-costs=[157]
+| | 04:SCAN HDFS [tpcds_partitioned_parquet_snap.store, RANDOM]
+| | HDFS partitions=1/1 files=1 size=119.76KB
+| | predicates: s_state = 'TN'
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store.s_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store.s_store_sk
+| | stored statistics:
+| | table: rows=1.35K size=119.76KB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=1.35K
+| | parquet statistics predicates: s_state = 'TN'
+| | parquet dictionary predicates: s_state = 'TN'
+| | mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
+| | tuple-ids=4 row-size=18B cardinality=46 cost=154
+| | in pipelines: 04(GETNEXT)
+| |
+| 17:AGGREGATE [FINALIZE]
+| | output: sum:merge(SR_FEE)
+| | group by: sr_customer_sk, sr_store_sk
+| | mem-estimate=55.21MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=2 row-size=24B cardinality=160.81M cost=949210422
+| | in pipelines: 17(GETNEXT), 00(OPEN)
+| |
+| 16:EXCHANGE [HASH(sr_customer_sk,sr_store_sk)]
+| | mem-estimate=13.01MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=2 row-size=24B cardinality=160.81M cost=35619914
+| | in pipelines: 00(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=110 (adjusted from 120)
+| Per-Host Shared Resources: mem-estimate=3.00MB mem-reservation=3.00MB thread-reservation=0 runtime-filters-memory=3.00MB
+| Per-Instance Resources: mem-estimate=64.58MB mem-reservation=34.50MB thread-reservation=1
+| max-parallelism=110 segment-costs=[1056649089, 331401897] cpu-comparison-result=110 [max(110 (self) vs 11 (sum children))]
+| 03:AGGREGATE [STREAMING]
+| | output: sum(SR_FEE)
+| | group by: sr_customer_sk, sr_store_sk
+| | mem-estimate=37.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=2 row-size=24B cardinality=160.81M cost=949210422
+| | in pipelines: 00(GETNEXT)
+| |
+| 02:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=04
+| | hash predicates: sr_returned_date_sk = d_date_sk
+| | fk/pk conjuncts: sr_returned_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=0,1 row-size=24B cardinality=160.81M cost=70387524
+| | in pipelines: 00(GETNEXT), 01(OPEN)
+| |
+| |--F15:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=22.33MB mem-reservation=22.31MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=04 plan-id=05 cohort-id=03
+| | | build expressions: d_date_sk
+| | | runtime filters: RF006[bloom] <- d_date_sk
+| | | mem-estimate=21.31MB mem-reservation=21.31MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 15:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=1 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 01(GETNEXT)
+| | |
+| | F02:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 28:TUPLE CACHE
+| | | cache key: 5536b7721c3902915addf658472e0caa
+| | | input scan node ids: 1
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=1 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 01(GETNEXT)
+| | |
+| | 01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=1 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 01(GETNEXT)
+| |
+| 00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns, RANDOM]
+| HDFS partitions=2004/2004 files=2004 size=48.14GB
+| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF005[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF006[bloom] -> sr_returned_date_sk
+| stored statistics:
+| table: rows=863.99M size=48.14GB
+| partitions: 2004/2004 rows=863.99M
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=30.09M est-scan-range=373(filtered from 2004)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=0 row-size=16B cardinality=160.81M(filtered from 863.99M) cost=37051143
+| in pipelines: 00(GETNEXT)
+|
+19:EXCHANGE [HASH(c_customer_sk)]
+| mem-estimate=10.35MB mem-reservation=0B thread-reservation=0
+| tuple-ids=5 row-size=32B cardinality=11.07M(filtered from 30.00M) cost=2858901
+| in pipelines: 05(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+Per-Host Shared Resources: mem-estimate=8.00MB mem-reservation=8.00MB thread-reservation=0 runtime-filters-memory=8.00MB
+Per-Instance Resources: mem-estimate=18.81MB mem-reservation=1.00MB thread-reservation=1
+max-parallelism=10 segment-costs=[37151541]
+05:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+ HDFS partitions=1/1 files=1 size=1.55GB
+ runtime filters: RF003[min_max] -> c_customer_sk, RF002[bloom] -> c_customer_sk
+ stored statistics:
+ table: rows=30.00M size=1.55GB
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=3.10M
+ mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+ tuple-ids=5 row-size=32B cardinality=11.07M(filtered from 30.00M) cost=8640000
+ in pipelines: 05(GETNEXT)
+====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q02.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q02.test
new file mode 100644
index 000000000..3ad637cdc
--- /dev/null
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q02.test
@@ -0,0 +1,1000 @@
+# TPCDS-Q02
+# start query 2 in stream 0 using template query2.tpl using seed 1930872976
+with wscs as
+ (select sold_date_sk
+ ,sales_price
+ from (select ws_sold_date_sk sold_date_sk
+ ,ws_ext_sales_price sales_price
+ from web_sales
+ union all
+ select cs_sold_date_sk sold_date_sk
+ ,cs_ext_sales_price sales_price
+ from catalog_sales) x),
+ wswscs as
+ (select d_week_seq,
+ sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales,
+ sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales,
+ sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales,
+ sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales,
+ sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales,
+ sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales,
+ sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales
+ from wscs
+ ,date_dim
+ where d_date_sk = sold_date_sk
+ group by d_week_seq)
+ select d_week_seq1
+ ,round(sun_sales1/sun_sales2,2)
+ ,round(mon_sales1/mon_sales2,2)
+ ,round(tue_sales1/tue_sales2,2)
+ ,round(wed_sales1/wed_sales2,2)
+ ,round(thu_sales1/thu_sales2,2)
+ ,round(fri_sales1/fri_sales2,2)
+ ,round(sat_sales1/sat_sales2,2)
+ from
+ (select wswscs.d_week_seq d_week_seq1
+ ,sun_sales sun_sales1
+ ,mon_sales mon_sales1
+ ,tue_sales tue_sales1
+ ,wed_sales wed_sales1
+ ,thu_sales thu_sales1
+ ,fri_sales fri_sales1
+ ,sat_sales sat_sales1
+ from wswscs,date_dim
+ where date_dim.d_week_seq = wswscs.d_week_seq and
+ d_year = 1998) y,
+ (select wswscs.d_week_seq d_week_seq2
+ ,sun_sales sun_sales2
+ ,mon_sales mon_sales2
+ ,tue_sales tue_sales2
+ ,wed_sales wed_sales2
+ ,thu_sales thu_sales2
+ ,fri_sales fri_sales2
+ ,sat_sales sat_sales2
+ from wswscs
+ ,date_dim
+ where date_dim.d_week_seq = wswscs.d_week_seq and
+ d_year = 1998+1) z
+ where d_week_seq1=d_week_seq2-53
+ order by d_week_seq1;
+
+# end query 2 in stream 0 using template query2.tpl
+---- PLAN
+Max Per-Host Resource Reservation: Memory=54.62MB Threads=1
+Per-Host Resource Estimates: Memory=142MB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=142.38MB mem-reservation=54.62MB thread-reservation=1 runtime-filters-memory=5.00MB
+| max-parallelism=1 segment-costs=[8199966861, 8199966861, 58178, 20480]
+PLAN-ROOT SINK
+| output exprs: d_week_seq1, round(sun_sales1 / sun_sales2, CAST(2 AS TINYINT)), round(mon_sales1 / mon_sales2, CAST(2 AS TINYINT)), round(tue_sales1 / tue_sales2, CAST(2 AS TINYINT)), round(wed_sales1 / wed_sales2, CAST(2 AS TINYINT)), round(thu_sales1 / thu_sales2, CAST(2 AS TINYINT)), round(fri_sales1 / fri_sales2, CAST(2 AS TINYINT)), round(sat_sales1 / sat_sales2, CAST(2 AS TINYINT))
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=20480
+|
+17:SORT
+| order by: d_week_seq1 ASC
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=20 row-size=228B cardinality=2.56K cost=26432
+| in pipelines: 17(GETNEXT), 05(OPEN)
+|
+24:TUPLE CACHE
+| cache key: 7e01fcabe46f2928038bce908db571f3
+| input scan node ids: 1,2,3,6,9,10,11,14
+| estimated serialized size: 660.00KB
+| estimated serialized size per node: 66.00KB
+| cumulative processing cost: 16399965468
+| cache read processing cost: 340
+| cache write processing cost: 1824
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=6,8,16,18 row-size=248B cardinality=2.56K cost=0
+| in pipelines: 05(GETNEXT)
+|
+16:HASH JOIN [INNER JOIN]
+| hash predicates: d_week_seq = d_week_seq - 53
+| fk/pk conjuncts: assumed fk/pk
+| runtime filters: RF000[bloom] <- d_week_seq - 53, RF001[min_max] <- d_week_seq - 53
+| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=6,8,16,18 row-size=248B cardinality=2.56K cost=3680
+| in pipelines: 05(GETNEXT), 13(OPEN)
+|
+|--23:TUPLE CACHE
+| | cache key: 3a362f84d4f9ad1d962a7d81b3fa5d00
+| | input scan node ids: 9,10,11,14
+| | estimated serialized size: 330.00KB
+| | estimated serialized size per node: 33.00KB
+| | cumulative processing cost: 8199980894
+| | cache read processing cost: 340
+| | cache write processing cost: 912
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=16,18 row-size=124B cardinality=2.56K cost=0
+| | in pipelines: 13(GETNEXT)
+| |
+| 15:HASH JOIN [INNER JOIN]
+| | hash predicates: d_week_seq = date_dim.d_week_seq
+| | fk/pk conjuncts: none
+| | runtime filters: RF006[bloom] <- date_dim.d_week_seq, RF007[min_max] <- date_dim.d_week_seq
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=16,18 row-size=124B cardinality=2.56K cost=3566
+| | in pipelines: 13(GETNEXT), 14(OPEN)
+| |
+| |--22:TUPLE CACHE
+| | | cache key: 466a66a56234a8dbff66a87cbb69c3a0
+| | | input scan node ids: 14
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=18 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 14(GETNEXT)
+| | |
+| | 14:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: d_year = CAST(1999 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: d_year = CAST(1999 AS INT)
+| | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| | tuple-ids=18 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 14(GETNEXT)
+| |
+| 21:TUPLE CACHE
+| | cache key: ee03d8782427908e37a8923be450ca2a
+| | input scan node ids: 9,10,11
+| | estimated serialized size: 1.22MB
+| | estimated serialized size per node: 124.71KB
+| | cumulative processing cost: 8199966861
+| | cache read processing cost: 1414
+| | cache write processing cost: 3448
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=16 row-size=116B cardinality=10.64K cost=0
+| | in pipelines: 13(GETNEXT)
+| |
+| 13:AGGREGATE [FINALIZE]
+| | output: sum(CASE WHEN (d_day_name = 'Sunday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Monday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Tuesday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Wednesday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Thursday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Friday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Saturday') THEN s [...]
+| | group by: d_week_seq
+| | mem-estimate=10.00MB mem-reservation=2.88MB spill-buffer=128.00KB thread-reservation=0
+| | tuple-ids=16 row-size=116B cardinality=10.64K cost=4617060482
+| | in pipelines: 13(GETNEXT), 09(OPEN), 10(OPEN)
+| |
+| 12:HASH JOIN [INNER JOIN]
+| | hash predicates: sold_date_sk = d_date_sk
+| | fk/pk conjuncts: assumed fk/pk
+| | runtime filters: RF008[bloom] <- d_date_sk
+| | mem-estimate=4.75MB mem-reservation=4.75MB spill-buffer=256.00KB thread-reservation=0
+| | tuple-ids=12,15 row-size=35B cardinality=6.48G cost=2836389159
+| | in pipelines: 09(GETNEXT), 10(GETNEXT), 11(OPEN)
+| |
+| |--11:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF006[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=15 row-size=27B cardinality=73.05K cost=15928
+| | in pipelines: 11(GETNEXT)
+| |
+| 08:UNION
+| | pass-through-operands: all
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=12 row-size=8B cardinality=6.48G cost=0
+| | in pipelines: 09(GETNEXT), 10(GETNEXT)
+| |
+| |--10:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales]
+| | HDFS partitions=1831/1831 files=1831 size=280.96GB
+| | deterministic scan range assignment: true
+| | runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
+| | stored statistics:
+| | table: rows=4.32G size=280.96GB
+| | partitions: 1831/1831 rows=4.32G
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=21.52M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=11 row-size=8B cardinality=4.32G cost=497672983
+| | in pipelines: 10(GETNEXT)
+| |
+| 09:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
+| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| deterministic scan range assignment: true
+| runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
+| stored statistics:
+| table: rows=2.16G size=145.75GB
+| partitions: 1824/1824 rows=2.16G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.37M
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=10 row-size=8B cardinality=2.16G cost=248828309
+| in pipelines: 09(GETNEXT)
+|
+20:TUPLE CACHE
+| cache key: 9aef10ab092234b8cbca9c113b466a31
+| input scan node ids: 1,2,3,6
+| estimated serialized size: 330.00KB
+| estimated serialized size per node: 33.00KB
+| cumulative processing cost: 8199980894
+| cache read processing cost: 340
+| cache write processing cost: 912
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=6,8 row-size=124B cardinality=2.56K cost=0
+| in pipelines: 05(GETNEXT)
+|
+07:HASH JOIN [INNER JOIN]
+| hash predicates: d_week_seq = date_dim.d_week_seq
+| fk/pk conjuncts: none
+| runtime filters: RF002[bloom] <- date_dim.d_week_seq, RF003[min_max] <- date_dim.d_week_seq
+| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=6,8 row-size=124B cardinality=2.56K cost=3566
+| in pipelines: 05(GETNEXT), 06(OPEN)
+|
+|--19:TUPLE CACHE
+| | cache key: 338f3f4d6b462b91bb22344145a15f28
+| | input scan node ids: 6
+| | estimated serialized size: 4.37KB
+| | estimated serialized size per node: 4.37KB
+| | cumulative processing cost: 10467
+| | cache read processing cost: 49
+| | cache write processing cost: 12
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=8 row-size=8B cardinality=373 cost=0
+| | in pipelines: 06(GETNEXT)
+| |
+| 06:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| HDFS partitions=1/1 files=1 size=2.17MB
+| deterministic scan range assignment: true
+| predicates: d_year = CAST(1998 AS INT)
+| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF000[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq
+| stored statistics:
+| table: rows=73.05K size=2.17MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet statistics predicates: d_year = CAST(1998 AS INT)
+| parquet dictionary predicates: d_year = CAST(1998 AS INT)
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=8 row-size=8B cardinality=373 cost=10467
+| in pipelines: 06(GETNEXT)
+|
+18:TUPLE CACHE
+| cache key: f548671f19cb71f02ec3a45d5ea26e74
+| input scan node ids: 1,2,3
+| estimated serialized size: 1.22MB
+| estimated serialized size per node: 124.71KB
+| cumulative processing cost: 8199966861
+| cache read processing cost: 1414
+| cache write processing cost: 3448
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=6 row-size=116B cardinality=10.64K cost=0
+| in pipelines: 05(GETNEXT)
+|
+05:AGGREGATE [FINALIZE]
+| output: sum(CASE WHEN (d_day_name = 'Sunday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Monday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Tuesday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Wednesday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Thursday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Friday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Saturday') THEN sale [...]
+| group by: d_week_seq
+| mem-estimate=10.00MB mem-reservation=2.88MB spill-buffer=128.00KB thread-reservation=0
+| tuple-ids=6 row-size=116B cardinality=10.64K cost=4617060482
+| in pipelines: 05(GETNEXT), 01(OPEN), 02(OPEN)
+|
+04:HASH JOIN [INNER JOIN]
+| hash predicates: sold_date_sk = d_date_sk
+| fk/pk conjuncts: assumed fk/pk
+| runtime filters: RF004[bloom] <- d_date_sk
+| mem-estimate=4.75MB mem-reservation=4.75MB spill-buffer=256.00KB thread-reservation=0
+| tuple-ids=2,5 row-size=35B cardinality=6.48G cost=2836389159
+| in pipelines: 01(GETNEXT), 02(GETNEXT), 03(OPEN)
+|
+|--03:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| HDFS partitions=1/1 files=1 size=2.17MB
+| deterministic scan range assignment: true
+| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF003[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF000[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF002[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq
+| stored statistics:
+| table: rows=73.05K size=2.17MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=5 row-size=27B cardinality=73.05K cost=15928
+| in pipelines: 03(GETNEXT)
+|
+00:UNION
+| pass-through-operands: all
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=2 row-size=8B cardinality=6.48G cost=0
+| in pipelines: 01(GETNEXT), 02(GETNEXT)
+|
+|--02:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales]
+| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| deterministic scan range assignment: true
+| runtime filters: RF004[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
+| stored statistics:
+| table: rows=4.32G size=280.96GB
+| partitions: 1831/1831 rows=4.32G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=21.52M
+| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| tuple-ids=1 row-size=8B cardinality=4.32G cost=497672983
+| in pipelines: 02(GETNEXT)
+|
+01:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
+ HDFS partitions=1824/1824 files=1824 size=145.75GB
+ deterministic scan range assignment: true
+ runtime filters: RF004[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
+ stored statistics:
+ table: rows=2.16G size=145.75GB
+ partitions: 1824/1824 rows=2.16G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=3.37M
+ mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+ tuple-ids=0 row-size=8B cardinality=2.16G cost=248828309
+ in pipelines: 01(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=294.19MB Threads=36
+Per-Host Resource Estimates: Memory=1.04GB
+F12:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=6.32MB mem-reservation=4.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[22294] cpu-comparison-result=262 [max(1 (self) vs 262 (sum children))]
+PLAN-ROOT SINK
+| output exprs: d_week_seq1, round(sun_sales1 / sun_sales2, CAST(2 AS TINYINT)), round(mon_sales1 / mon_sales2, CAST(2 AS TINYINT)), round(tue_sales1 / tue_sales2, CAST(2 AS TINYINT)), round(wed_sales1 / wed_sales2, CAST(2 AS TINYINT)), round(thu_sales1 / thu_sales2, CAST(2 AS TINYINT)), round(fri_sales1 / fri_sales2, CAST(2 AS TINYINT)), round(sat_sales1 / sat_sales2, CAST(2 AS TINYINT))
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=20480
+|
+27:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: d_week_seq1 ASC
+| mem-estimate=2.32MB mem-reservation=0B thread-reservation=0
+| tuple-ids=20 row-size=228B cardinality=2.56K cost=1814
+| in pipelines: 17(GETNEXT)
+|
+F04:PLAN FRAGMENT [HASH(d_week_seq)] hosts=10 instances=10 (adjusted from 120)
+Per-Instance Resources: mem-estimate=34.06MB mem-reservation=7.94MB thread-reservation=1
+max-parallelism=10 segment-costs=[1760943, 30745, 16097] cpu-comparison-result=262 [max(10 (self) vs 262 (sum children))]
+17:SORT
+| order by: d_week_seq1 ASC
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=20 row-size=228B cardinality=2.56K cost=26432
+| in pipelines: 17(GETNEXT), 20(OPEN)
+|
+16:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=00
+| hash predicates: d_week_seq = d_week_seq - 53
+| fk/pk conjuncts: assumed fk/pk
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=6,8,16,18 row-size=248B cardinality=2.56K cost=1120
+| in pipelines: 20(GETNEXT), 24(OPEN)
+|
+|--F13:PLAN FRAGMENT [HASH(d_week_seq)] hosts=10 instances=10 (adjusted from 120)
+| | Per-Instance Resources: mem-estimate=4.26MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[4398] cpu-comparison-result=131 [max(10 (self) vs 131 (sum children))]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: d_week_seq - 53
+| | runtime filters: RF000[bloom] <- d_week_seq - 53, RF001[min_max] <- d_week_seq - 53
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0 cost=2560
+| |
+| 26:EXCHANGE [HASH(d_week_seq - 53)]
+| | mem-estimate=1.32MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=16,18 row-size=124B cardinality=2.56K cost=1838
+| | in pipelines: 24(GETNEXT)
+| |
+| F10:PLAN FRAGMENT [HASH(d_week_seq)] hosts=10 instances=10 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=34.06MB mem-reservation=1.94MB thread-reservation=1
+| max-parallelism=10 segment-costs=[1760943, 26273] cpu-comparison-result=131 [max(10 (self) vs 131 (sum children))]
+| 15:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=01
+| | hash predicates: d_week_seq = date_dim.d_week_seq
+| | fk/pk conjuncts: none
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=16,18 row-size=124B cardinality=2.56K cost=3193
+| | in pipelines: 24(GETNEXT), 14(OPEN)
+| |
+| |--F14:PLAN FRAGMENT [HASH(d_week_seq)] hosts=10 instances=10 (adjusted from 120)
+| | | Per-Instance Resources: mem-estimate=2.95MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[428]
+| | JOIN BUILD
+| | | join-table-id=01 plan-id=02 cohort-id=02
+| | | build expressions: date_dim.d_week_seq
+| | | runtime filters: RF006[bloom] <- date_dim.d_week_seq, RF007[min_max] <- date_dim.d_week_seq
+| | | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 25:EXCHANGE [HASH(date_dim.d_week_seq)]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=18 row-size=8B cardinality=373 cost=55
+| | | in pipelines: 14(GETNEXT)
+| | |
+| | F11:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.47MB mem-reservation=4.25MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10851]
+| | 29:TUPLE CACHE
+| | | cache key: 466a66a56234a8dbff66a87cbb69c3a0
+| | | input scan node ids: 14
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=18 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 14(GETNEXT)
+| | |
+| | 14:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: d_year = CAST(1999 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: d_year = CAST(1999 AS INT)
+| | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| | tuple-ids=18 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 14(GETNEXT)
+| |
+| 24:AGGREGATE [FINALIZE]
+| | output: sum:merge(CASE WHEN (d_day_name = 'Sunday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Monday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Tuesday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Wednesday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Thursday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Friday') THEN sales_price ELSE NULL END), sum:merge( [...]
+| | group by: d_week_seq
+| | mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=16 row-size=116B cardinality=10.64K cost=937636
+| | in pipelines: 24(GETNEXT), 09(OPEN), 10(OPEN)
+| |
+| 23:EXCHANGE [HASH(d_week_seq)]
+| | mem-estimate=24.06MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=16 row-size=116B cardinality=1.28M cost=823307
+| | in pipelines: 09(GETNEXT), 10(GETNEXT)
+| |
+| F08:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| Per-Instance Resources: mem-estimate=34.69MB mem-reservation=8.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[8203179637, 10197930] cpu-comparison-result=120 [max(120 (self) vs 11 (sum children))]
+| 28:TUPLE CACHE
+| | cache key: 8ed4943a1c1e05ec1e038827ab4ea50e
+| | input scan node ids: 9,10
+| | estimated serialized size: 146.15MB
+| | estimated serialized size per node: 14.61MB
+| | cumulative processing cost: 8203372625
+| | cache read processing cost: 169718
+| | cache write processing cost: 413760
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=16 row-size=116B cardinality=1.28M cost=0
+| | in pipelines: 09(GETNEXT), 10(GETNEXT)
+| |
+| 13:AGGREGATE [STREAMING]
+| | output: sum(CASE WHEN (d_day_name = 'Sunday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Monday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Tuesday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Wednesday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Thursday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Friday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Saturday') THEN s [...]
+| | group by: d_week_seq
+| | mem-estimate=10.00MB mem-reservation=3.00MB spill-buffer=128.00KB thread-reservation=0
+| | tuple-ids=16 row-size=116B cardinality=1.28M cost=4620362235
+| | in pipelines: 09(GETNEXT), 10(GETNEXT)
+| |
+| 12:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=02
+| | hash predicates: sold_date_sk = d_date_sk
+| | fk/pk conjuncts: assumed fk/pk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=256.00KB thread-reservation=0
+| | tuple-ids=12,15 row-size=35B cardinality=6.48G cost=2836316110
+| | in pipelines: 09(GETNEXT), 10(GETNEXT), 11(OPEN)
+| |
+| |--F15:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=59.92MB mem-reservation=58.00MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[170129]
+| | JOIN BUILD
+| | | join-table-id=02 plan-id=03 cohort-id=02
+| | | build expressions: d_date_sk
+| | | runtime filters: RF008[bloom] <- d_date_sk
+| | | mem-estimate=57.00MB mem-reservation=57.00MB spill-buffer=256.00KB thread-reservation=0 cost=73049
+| | |
+| | 22:EXCHANGE [BROADCAST]
+| | | mem-estimate=1.92MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=15 row-size=27B cardinality=73.05K cost=97080
+| | | in pipelines: 11(GETNEXT)
+| | |
+| | F09:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| | Per-Instance Resources: mem-estimate=16.12MB mem-reservation=512.00KB thread-reservation=1
+| | max-parallelism=1 segment-costs=[22859]
+| | 11:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF006[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=15 row-size=27B cardinality=73.05K cost=15928
+| | in pipelines: 11(GETNEXT)
+| |
+| 08:UNION
+| | pass-through-operands: all
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=12 row-size=8B cardinality=6.48G cost=0
+| | in pipelines: 09(GETNEXT), 10(GETNEXT)
+| |
+| |--10:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
+| | HDFS partitions=1831/1831 files=1831 size=280.96GB
+| | deterministic scan range assignment: true
+| | runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
+| | stored statistics:
+| | table: rows=4.32G size=280.96GB
+| | partitions: 1831/1831 rows=4.32G
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=21.52M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=11 row-size=8B cardinality=4.32G cost=497672983
+| | in pipelines: 10(GETNEXT)
+| |
+| 09:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
+| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| deterministic scan range assignment: true
+| runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
+| stored statistics:
+| table: rows=2.16G size=145.75GB
+| partitions: 1824/1824 rows=2.16G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.37M
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=10 row-size=8B cardinality=2.16G cost=248828309
+| in pipelines: 09(GETNEXT)
+|
+07:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=03
+| hash predicates: d_week_seq = date_dim.d_week_seq
+| fk/pk conjuncts: none
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=6,8 row-size=124B cardinality=2.56K cost=3193
+| in pipelines: 20(GETNEXT), 06(OPEN)
+|
+|--F16:PLAN FRAGMENT [HASH(d_week_seq)] hosts=10 instances=10 (adjusted from 120)
+| | Per-Instance Resources: mem-estimate=2.95MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[428]
+| JOIN BUILD
+| | join-table-id=03 plan-id=04 cohort-id=01
+| | build expressions: date_dim.d_week_seq
+| | runtime filters: RF002[bloom] <- date_dim.d_week_seq, RF003[min_max] <- date_dim.d_week_seq
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| |
+| 21:EXCHANGE [HASH(date_dim.d_week_seq)]
+| | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=8 row-size=8B cardinality=373 cost=55
+| | in pipelines: 06(GETNEXT)
+| |
+| F05:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| Per-Instance Resources: mem-estimate=16.47MB mem-reservation=256.00KB thread-reservation=1
+| max-parallelism=1 segment-costs=[10851]
+| 06:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| HDFS partitions=1/1 files=1 size=2.17MB
+| predicates: d_year = CAST(1998 AS INT)
+| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF000[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq
+| stored statistics:
+| table: rows=73.05K size=2.17MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet statistics predicates: d_year = CAST(1998 AS INT)
+| parquet dictionary predicates: d_year = CAST(1998 AS INT)
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=8 row-size=8B cardinality=373 cost=10467
+| in pipelines: 06(GETNEXT)
+|
+20:AGGREGATE [FINALIZE]
+| output: sum:merge(CASE WHEN (d_day_name = 'Sunday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Monday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Tuesday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Wednesday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Thursday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Friday') THEN sales_price ELSE NULL END), sum:merge(CAS [...]
+| group by: d_week_seq
+| mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=6 row-size=116B cardinality=10.64K cost=937636
+| in pipelines: 20(GETNEXT), 01(OPEN), 02(OPEN)
+|
+19:EXCHANGE [HASH(d_week_seq)]
+| mem-estimate=24.06MB mem-reservation=0B thread-reservation=0
+| tuple-ids=6 row-size=116B cardinality=1.28M cost=823307
+| in pipelines: 01(GETNEXT), 02(GETNEXT)
+|
+F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+Per-Instance Resources: mem-estimate=30.69MB mem-reservation=4.00MB thread-reservation=1
+max-parallelism=120 segment-costs=[8203179637, 10197930] cpu-comparison-result=120 [max(120 (self) vs 11 (sum children))]
+05:AGGREGATE [STREAMING]
+| output: sum(CASE WHEN (d_day_name = 'Sunday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Monday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Tuesday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Wednesday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Thursday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Friday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Saturday') THEN sale [...]
+| group by: d_week_seq
+| mem-estimate=10.00MB mem-reservation=3.00MB spill-buffer=128.00KB thread-reservation=0
+| tuple-ids=6 row-size=116B cardinality=1.28M cost=4620362235
+| in pipelines: 01(GETNEXT), 02(GETNEXT)
+|
+04:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=04
+| hash predicates: sold_date_sk = d_date_sk
+| fk/pk conjuncts: assumed fk/pk
+| mem-estimate=0B mem-reservation=0B spill-buffer=256.00KB thread-reservation=0
+| tuple-ids=2,5 row-size=35B cardinality=6.48G cost=2836316110
+| in pipelines: 01(GETNEXT), 02(GETNEXT), 03(OPEN)
+|
+|--F17:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=59.92MB mem-reservation=58.00MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[170129]
+| JOIN BUILD
+| | join-table-id=04 plan-id=05 cohort-id=01
+| | build expressions: d_date_sk
+| | runtime filters: RF004[bloom] <- d_date_sk
+| | mem-estimate=57.00MB mem-reservation=57.00MB spill-buffer=256.00KB thread-reservation=0 cost=73049
+| |
+| 18:EXCHANGE [BROADCAST]
+| | mem-estimate=1.92MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=5 row-size=27B cardinality=73.05K cost=97080
+| | in pipelines: 03(GETNEXT)
+| |
+| F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
+| Per-Instance Resources: mem-estimate=16.12MB mem-reservation=512.00KB thread-reservation=1
+| max-parallelism=1 segment-costs=[22859]
+| 03:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| HDFS partitions=1/1 files=1 size=2.17MB
+| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF003[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF000[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF002[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq
+| stored statistics:
+| table: rows=73.05K size=2.17MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=5 row-size=27B cardinality=73.05K cost=15928
+| in pipelines: 03(GETNEXT)
+|
+00:UNION
+| pass-through-operands: all
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=2 row-size=8B cardinality=6.48G cost=0
+| in pipelines: 01(GETNEXT), 02(GETNEXT)
+|
+|--02:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
+| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| runtime filters: RF004[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
+| stored statistics:
+| table: rows=4.32G size=280.96GB
+| partitions: 1831/1831 rows=4.32G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=21.52M
+| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| tuple-ids=1 row-size=8B cardinality=4.32G cost=497672983
+| in pipelines: 02(GETNEXT)
+|
+01:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=145.75GB
+ runtime filters: RF004[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
+ stored statistics:
+ table: rows=2.16G size=145.75GB
+ partitions: 1824/1824 rows=2.16G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=3.37M
+ mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+ tuple-ids=0 row-size=8B cardinality=2.16G cost=248828309
+ in pipelines: 01(GETNEXT)
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=294.19MB Threads=36
+Per-Host Resource Estimates: Memory=1.04GB
+F12:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=6.32MB mem-reservation=4.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[22294] cpu-comparison-result=262 [max(1 (self) vs 262 (sum children))]
+PLAN-ROOT SINK
+| output exprs: d_week_seq1, round(sun_sales1 / sun_sales2, CAST(2 AS TINYINT)), round(mon_sales1 / mon_sales2, CAST(2 AS TINYINT)), round(tue_sales1 / tue_sales2, CAST(2 AS TINYINT)), round(wed_sales1 / wed_sales2, CAST(2 AS TINYINT)), round(thu_sales1 / thu_sales2, CAST(2 AS TINYINT)), round(fri_sales1 / fri_sales2, CAST(2 AS TINYINT)), round(sat_sales1 / sat_sales2, CAST(2 AS TINYINT))
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=20480
+|
+27:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: d_week_seq1 ASC
+| mem-estimate=2.32MB mem-reservation=0B thread-reservation=0
+| tuple-ids=20 row-size=228B cardinality=2.56K cost=1814
+| in pipelines: 17(GETNEXT)
+|
+F04:PLAN FRAGMENT [HASH(d_week_seq)] hosts=10 instances=10 (adjusted from 120)
+Per-Instance Resources: mem-estimate=34.06MB mem-reservation=7.94MB thread-reservation=1
+max-parallelism=10 segment-costs=[1760943, 30745, 16097] cpu-comparison-result=262 [max(10 (self) vs 262 (sum children))]
+17:SORT
+| order by: d_week_seq1 ASC
+| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=20 row-size=228B cardinality=2.56K cost=26432
+| in pipelines: 17(GETNEXT), 20(OPEN)
+|
+16:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=00
+| hash predicates: d_week_seq = d_week_seq - 53
+| fk/pk conjuncts: assumed fk/pk
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=6,8,16,18 row-size=248B cardinality=2.56K cost=1120
+| in pipelines: 20(GETNEXT), 24(OPEN)
+|
+|--F13:PLAN FRAGMENT [HASH(d_week_seq)] hosts=10 instances=10 (adjusted from 120)
+| | Per-Instance Resources: mem-estimate=4.26MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[4398] cpu-comparison-result=131 [max(10 (self) vs 131 (sum children))]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: d_week_seq - 53
+| | runtime filters: RF000[bloom] <- d_week_seq - 53, RF001[min_max] <- d_week_seq - 53
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0 cost=2560
+| |
+| 26:EXCHANGE [HASH(d_week_seq - 53)]
+| | mem-estimate=1.32MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=16,18 row-size=124B cardinality=2.56K cost=1838
+| | in pipelines: 24(GETNEXT)
+| |
+| F10:PLAN FRAGMENT [HASH(d_week_seq)] hosts=10 instances=10 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=34.06MB mem-reservation=1.94MB thread-reservation=1
+| max-parallelism=10 segment-costs=[1760943, 26273] cpu-comparison-result=131 [max(10 (self) vs 131 (sum children))]
+| 15:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=01
+| | hash predicates: d_week_seq = date_dim.d_week_seq
+| | fk/pk conjuncts: none
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=16,18 row-size=124B cardinality=2.56K cost=3193
+| | in pipelines: 24(GETNEXT), 14(OPEN)
+| |
+| |--F14:PLAN FRAGMENT [HASH(d_week_seq)] hosts=10 instances=10 (adjusted from 120)
+| | | Per-Instance Resources: mem-estimate=2.95MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[428]
+| | JOIN BUILD
+| | | join-table-id=01 plan-id=02 cohort-id=02
+| | | build expressions: date_dim.d_week_seq
+| | | runtime filters: RF006[bloom] <- date_dim.d_week_seq, RF007[min_max] <- date_dim.d_week_seq
+| | | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 25:EXCHANGE [HASH(date_dim.d_week_seq)]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=18 row-size=8B cardinality=373 cost=55
+| | | in pipelines: 14(GETNEXT)
+| | |
+| | F11:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.47MB mem-reservation=4.25MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10851]
+| | 29:TUPLE CACHE
+| | | cache key: 466a66a56234a8dbff66a87cbb69c3a0
+| | | input scan node ids: 14
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=18 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 14(GETNEXT)
+| | |
+| | 14:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: d_year = CAST(1999 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: d_year = CAST(1999 AS INT)
+| | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| | tuple-ids=18 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 14(GETNEXT)
+| |
+| 24:AGGREGATE [FINALIZE]
+| | output: sum:merge(CASE WHEN (d_day_name = 'Sunday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Monday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Tuesday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Wednesday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Thursday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Friday') THEN sales_price ELSE NULL END), sum:merge( [...]
+| | group by: d_week_seq
+| | mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=16 row-size=116B cardinality=10.64K cost=937636
+| | in pipelines: 24(GETNEXT), 09(OPEN), 10(OPEN)
+| |
+| 23:EXCHANGE [HASH(d_week_seq)]
+| | mem-estimate=24.06MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=16 row-size=116B cardinality=1.28M cost=823307
+| | in pipelines: 09(GETNEXT), 10(GETNEXT)
+| |
+| F08:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| Per-Instance Resources: mem-estimate=34.69MB mem-reservation=8.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[8203179637, 10197930] cpu-comparison-result=120 [max(120 (self) vs 11 (sum children))]
+| 28:TUPLE CACHE
+| | cache key: 8ed4943a1c1e05ec1e038827ab4ea50e
+| | input scan node ids: 9,10
+| | estimated serialized size: 146.15MB
+| | estimated serialized size per node: 14.61MB
+| | cumulative processing cost: 8203372625
+| | cache read processing cost: 169718
+| | cache write processing cost: 413760
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=16 row-size=116B cardinality=1.28M cost=0
+| | in pipelines: 09(GETNEXT), 10(GETNEXT)
+| |
+| 13:AGGREGATE [STREAMING]
+| | output: sum(CASE WHEN (d_day_name = 'Sunday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Monday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Tuesday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Wednesday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Thursday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Friday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Saturday') THEN s [...]
+| | group by: d_week_seq
+| | mem-estimate=10.00MB mem-reservation=3.00MB spill-buffer=128.00KB thread-reservation=0
+| | tuple-ids=16 row-size=116B cardinality=1.28M cost=4620362235
+| | in pipelines: 09(GETNEXT), 10(GETNEXT)
+| |
+| 12:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=02
+| | hash predicates: sold_date_sk = d_date_sk
+| | fk/pk conjuncts: assumed fk/pk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=256.00KB thread-reservation=0
+| | tuple-ids=12,15 row-size=35B cardinality=6.48G cost=2836316110
+| | in pipelines: 09(GETNEXT), 10(GETNEXT), 11(OPEN)
+| |
+| |--F15:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=59.92MB mem-reservation=58.00MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[170129]
+| | JOIN BUILD
+| | | join-table-id=02 plan-id=03 cohort-id=02
+| | | build expressions: d_date_sk
+| | | runtime filters: RF008[bloom] <- d_date_sk
+| | | mem-estimate=57.00MB mem-reservation=57.00MB spill-buffer=256.00KB thread-reservation=0 cost=73049
+| | |
+| | 22:EXCHANGE [BROADCAST]
+| | | mem-estimate=1.92MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=15 row-size=27B cardinality=73.05K cost=97080
+| | | in pipelines: 11(GETNEXT)
+| | |
+| | F09:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| | Per-Instance Resources: mem-estimate=16.12MB mem-reservation=512.00KB thread-reservation=1
+| | max-parallelism=1 segment-costs=[22859]
+| | 11:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF006[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=15 row-size=27B cardinality=73.05K cost=15928
+| | in pipelines: 11(GETNEXT)
+| |
+| 08:UNION
+| | pass-through-operands: all
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=12 row-size=8B cardinality=6.48G cost=0
+| | in pipelines: 09(GETNEXT), 10(GETNEXT)
+| |
+| |--10:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
+| | HDFS partitions=1831/1831 files=1831 size=280.96GB
+| | deterministic scan range assignment: true
+| | runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
+| | stored statistics:
+| | table: rows=4.32G size=280.96GB
+| | partitions: 1831/1831 rows=4.32G
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=21.52M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=11 row-size=8B cardinality=4.32G cost=497672983
+| | in pipelines: 10(GETNEXT)
+| |
+| 09:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
+| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| deterministic scan range assignment: true
+| runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
+| stored statistics:
+| table: rows=2.16G size=145.75GB
+| partitions: 1824/1824 rows=2.16G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.37M
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=10 row-size=8B cardinality=2.16G cost=248828309
+| in pipelines: 09(GETNEXT)
+|
+07:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=03
+| hash predicates: d_week_seq = date_dim.d_week_seq
+| fk/pk conjuncts: none
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=6,8 row-size=124B cardinality=2.56K cost=3193
+| in pipelines: 20(GETNEXT), 06(OPEN)
+|
+|--F16:PLAN FRAGMENT [HASH(d_week_seq)] hosts=10 instances=10 (adjusted from 120)
+| | Per-Instance Resources: mem-estimate=2.95MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[428]
+| JOIN BUILD
+| | join-table-id=03 plan-id=04 cohort-id=01
+| | build expressions: date_dim.d_week_seq
+| | runtime filters: RF002[bloom] <- date_dim.d_week_seq, RF003[min_max] <- date_dim.d_week_seq
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| |
+| 21:EXCHANGE [HASH(date_dim.d_week_seq)]
+| | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=8 row-size=8B cardinality=373 cost=55
+| | in pipelines: 06(GETNEXT)
+| |
+| F05:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| Per-Instance Resources: mem-estimate=16.47MB mem-reservation=256.00KB thread-reservation=1
+| max-parallelism=1 segment-costs=[10851]
+| 06:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| HDFS partitions=1/1 files=1 size=2.17MB
+| predicates: d_year = CAST(1998 AS INT)
+| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF000[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq
+| stored statistics:
+| table: rows=73.05K size=2.17MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet statistics predicates: d_year = CAST(1998 AS INT)
+| parquet dictionary predicates: d_year = CAST(1998 AS INT)
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=8 row-size=8B cardinality=373 cost=10467
+| in pipelines: 06(GETNEXT)
+|
+20:AGGREGATE [FINALIZE]
+| output: sum:merge(CASE WHEN (d_day_name = 'Sunday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Monday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Tuesday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Wednesday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Thursday') THEN sales_price ELSE NULL END), sum:merge(CASE WHEN (d_day_name = 'Friday') THEN sales_price ELSE NULL END), sum:merge(CAS [...]
+| group by: d_week_seq
+| mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=6 row-size=116B cardinality=10.64K cost=937636
+| in pipelines: 20(GETNEXT), 01(OPEN), 02(OPEN)
+|
+19:EXCHANGE [HASH(d_week_seq)]
+| mem-estimate=24.06MB mem-reservation=0B thread-reservation=0
+| tuple-ids=6 row-size=116B cardinality=1.28M cost=823307
+| in pipelines: 01(GETNEXT), 02(GETNEXT)
+|
+F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+Per-Instance Resources: mem-estimate=30.69MB mem-reservation=4.00MB thread-reservation=1
+max-parallelism=120 segment-costs=[8203179637, 10197930] cpu-comparison-result=120 [max(120 (self) vs 11 (sum children))]
+05:AGGREGATE [STREAMING]
+| output: sum(CASE WHEN (d_day_name = 'Sunday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Monday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Tuesday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Wednesday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Thursday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Friday') THEN sales_price ELSE NULL END), sum(CASE WHEN (d_day_name = 'Saturday') THEN sale [...]
+| group by: d_week_seq
+| mem-estimate=10.00MB mem-reservation=3.00MB spill-buffer=128.00KB thread-reservation=0
+| tuple-ids=6 row-size=116B cardinality=1.28M cost=4620362235
+| in pipelines: 01(GETNEXT), 02(GETNEXT)
+|
+04:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=04
+| hash predicates: sold_date_sk = d_date_sk
+| fk/pk conjuncts: assumed fk/pk
+| mem-estimate=0B mem-reservation=0B spill-buffer=256.00KB thread-reservation=0
+| tuple-ids=2,5 row-size=35B cardinality=6.48G cost=2836316110
+| in pipelines: 01(GETNEXT), 02(GETNEXT), 03(OPEN)
+|
+|--F17:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=59.92MB mem-reservation=58.00MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[170129]
+| JOIN BUILD
+| | join-table-id=04 plan-id=05 cohort-id=01
+| | build expressions: d_date_sk
+| | runtime filters: RF004[bloom] <- d_date_sk
+| | mem-estimate=57.00MB mem-reservation=57.00MB spill-buffer=256.00KB thread-reservation=0 cost=73049
+| |
+| 18:EXCHANGE [BROADCAST]
+| | mem-estimate=1.92MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=5 row-size=27B cardinality=73.05K cost=97080
+| | in pipelines: 03(GETNEXT)
+| |
+| F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
+| Per-Instance Resources: mem-estimate=16.12MB mem-reservation=512.00KB thread-reservation=1
+| max-parallelism=1 segment-costs=[22859]
+| 03:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| HDFS partitions=1/1 files=1 size=2.17MB
+| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF003[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF000[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF002[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq
+| stored statistics:
+| table: rows=73.05K size=2.17MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=5 row-size=27B cardinality=73.05K cost=15928
+| in pipelines: 03(GETNEXT)
+|
+00:UNION
+| pass-through-operands: all
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=2 row-size=8B cardinality=6.48G cost=0
+| in pipelines: 01(GETNEXT), 02(GETNEXT)
+|
+|--02:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
+| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| runtime filters: RF004[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
+| stored statistics:
+| table: rows=4.32G size=280.96GB
+| partitions: 1831/1831 rows=4.32G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=21.52M
+| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| tuple-ids=1 row-size=8B cardinality=4.32G cost=497672983
+| in pipelines: 02(GETNEXT)
+|
+01:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=145.75GB
+ runtime filters: RF004[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
+ stored statistics:
+ table: rows=2.16G size=145.75GB
+ partitions: 1824/1824 rows=2.16G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=3.37M
+ mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+ tuple-ids=0 row-size=8B cardinality=2.16G cost=248828309
+ in pipelines: 01(GETNEXT)
+====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q03.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q03.test
new file mode 100644
index 000000000..aa3f464bb
--- /dev/null
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q03.test
@@ -0,0 +1,524 @@
+# TPCDS-Q3
+# start query 3 in stream 0 using template query3.tpl using seed 318176889
+select dt.d_year
+ ,item.i_brand_id brand_id
+ ,item.i_brand brand
+ ,sum(ss_sales_price) sum_agg
+ from date_dim dt
+ ,store_sales
+ ,item
+ where dt.d_date_sk = store_sales.ss_sold_date_sk
+ and store_sales.ss_item_sk = item.i_item_sk
+ and item.i_manufact_id = 816
+ and dt.d_moy=11
+ group by dt.d_year
+ ,item.i_brand
+ ,item.i_brand_id
+ order by dt.d_year
+ ,sum_agg desc
+ ,brand_id
+ limit 100;
+
+# end query 3 in stream 0 using template query3.tpl
+---- PLAN
+Max Per-Host Resource Reservation: Memory=55.88MB Threads=1
+Per-Host Resource Estimates: Memory=551MB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=551.48MB mem-reservation=55.88MB thread-reservation=1 runtime-filters-memory=2.00MB
+| max-parallelism=1 segment-costs=[1554129087, 117158429, 400]
+PLAN-ROOT SINK
+| output exprs: dt.d_year, item.i_brand_id, item.i_brand, sum(ss_sales_price)
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=400
+|
+06:TOP-N [LIMIT=100]
+| order by: dt.d_year ASC, sum(ss_sales_price) DESC, item.i_brand_id ASC
+| mem-estimate=5.09KB mem-reservation=0B thread-reservation=0
+| tuple-ids=4 row-size=52B cardinality=100 cost=117158429
+| in pipelines: 06(GETNEXT), 05(OPEN)
+|
+10:TUPLE CACHE
+| cache key: 88603377fd0f8f620cbe05663eb72182
+| input scan node ids: 1,2,0
+| estimated serialized size: 481.77MB
+| estimated serialized size per node: 48.18MB
+| cumulative processing cost: 1554129087
+| cache read processing cost: 1195493
+| cache write processing cost: 1363976
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=52B cardinality=9.00M cost=0
+| in pipelines: 05(GETNEXT)
+|
+05:AGGREGATE [FINALIZE]
+| output: sum(ss_sales_price)
+| group by: dt.d_year, item.i_brand, item.i_brand_id
+| mem-estimate=521.60MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=52B cardinality=9.00M cost=53096454
+| in pipelines: 05(GETNEXT), 01(OPEN)
+|
+04:HASH JOIN [INNER JOIN]
+| hash predicates: store_sales.ss_sold_date_sk = dt.d_date_sk
+| fk/pk conjuncts: store_sales.ss_sold_date_sk = dt.d_date_sk
+| runtime filters: RF000[bloom] <- dt.d_date_sk
+| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1,2,0 row-size=64B cardinality=9.00M cost=3943388
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+|--09:TUPLE CACHE
+| | cache key: c4355c97cf54825ec0d5f5fe4a27616e
+| | input scan node ids: 0
+| | estimated serialized size: 95.11KB
+| | estimated serialized size per node: 95.11KB
+| | cumulative processing cost: 14675
+| | cache read processing cost: 808
+| | cache write processing cost: 262
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=0 row-size=12B cardinality=6.09K cost=0
+| | in pipelines: 00(GETNEXT)
+| |
+| 00:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim dt]
+| HDFS partitions=1/1 files=1 size=2.17MB
+| deterministic scan range assignment: true
+| predicates: dt.d_moy = CAST(11 AS INT)
+| stored statistics:
+| table: rows=73.05K size=2.17MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet statistics predicates: dt.d_moy = CAST(11 AS INT)
+| parquet dictionary predicates: dt.d_moy = CAST(11 AS INT)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=0 row-size=12B cardinality=6.09K cost=14675
+| in pipelines: 00(GETNEXT)
+|
+03:HASH JOIN [INNER JOIN]
+| hash predicates: store_sales.ss_item_sk = item.i_item_sk
+| fk/pk conjuncts: store_sales.ss_item_sk = item.i_item_sk
+| runtime filters: RF002[bloom] <- item.i_item_sk, RF003[min_max] <- item.i_item_sk
+| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1,2 row-size=52B cardinality=9.00M cost=3937676
+| in pipelines: 01(GETNEXT), 02(OPEN)
+|
+|--08:TUPLE CACHE
+| | cache key: 9667419a8188dd9f26a93dc52053603b
+| | input scan node ids: 2
+| | estimated serialized size: 16.17KB
+| | estimated serialized size per node: 4.04KB
+| | cumulative processing cost: 156093
+| | cache read processing cost: 49
+| | cache write processing cost: 44
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=2 row-size=40B cardinality=375 cost=0
+| | in pipelines: 02(GETNEXT)
+| |
+| 02:SCAN HDFS [tpcds_partitioned_parquet_snap.item]
+| HDFS partitions=1/1 files=1 size=33.54MB
+| deterministic scan range assignment: true
+| predicates: item.i_manufact_id = CAST(816 AS INT)
+| stored statistics:
+| table: rows=360.00K size=33.54MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=101.54K
+| parquet statistics predicates: item.i_manufact_id = CAST(816 AS INT)
+| parquet dictionary predicates: item.i_manufact_id = CAST(816 AS INT)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=2 row-size=40B cardinality=375 cost=156093
+| in pipelines: 02(GETNEXT)
+|
+07:TUPLE CACHE
+| cache key: 9f4049df551e75dbc62bb8c7c80de917
+| input scan node ids: 1
+| estimated serialized size: 137.26MB
+| estimated serialized size per node: 13.73MB
+| cumulative processing cost: 1492980801
+| cache read processing cost: 1195493
+| cache write processing cost: 388602
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=12B cardinality=9.00M(filtered from 8.64G) cost=0
+| in pipelines: 01(GETNEXT)
+|
+01:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ deterministic scan range assignment: true
+ runtime filters: RF003[min_max] -> store_sales.ss_item_sk, RF000[bloom] -> store_sales.ss_sold_date_sk, RF002[bloom] -> store_sales.ss_item_sk
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=1 row-size=12B cardinality=9.00M(filtered from 8.64G) cost=1492980801
+ in pipelines: 01(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=383.50MB Threads=19
+Per-Host Resource Estimates: Memory=657MB
+F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=4.11MB mem-reservation=4.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[435] cpu-comparison-result=120 [max(1 (self) vs 120 (sum children))]
+PLAN-ROOT SINK
+| output exprs: dt.d_year, item.i_brand_id, item.i_brand, sum(ss_sales_price)
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=400
+|
+11:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: dt.d_year ASC, sum(ss_sales_price) DESC, item.i_brand_id ASC
+| limit: 100
+| mem-estimate=110.20KB mem-reservation=0B thread-reservation=0
+| tuple-ids=4 row-size=52B cardinality=100 cost=35
+| in pipelines: 06(GETNEXT)
+|
+F03:PLAN FRAGMENT [HASH(dt.d_year,item.i_brand,item.i_brand_id)] hosts=10 instances=20 (adjusted from 120)
+Per-Instance Resources: mem-estimate=50.58MB mem-reservation=34.00MB thread-reservation=1
+max-parallelism=20 segment-costs=[56254141, 117158429, 168] cpu-comparison-result=120 [max(20 (self) vs 120 (sum children))]
+06:TOP-N [LIMIT=100]
+| order by: dt.d_year ASC, sum(ss_sales_price) DESC, item.i_brand_id ASC
+| mem-estimate=5.09KB mem-reservation=0B thread-reservation=0
+| tuple-ids=4 row-size=52B cardinality=100 cost=117158429
+| in pipelines: 06(GETNEXT), 10(OPEN)
+|
+10:AGGREGATE [FINALIZE]
+| output: sum:merge(ss_sales_price)
+| group by: dt.d_year, item.i_brand, item.i_brand_id
+| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=52B cardinality=9.00M cost=53096454
+| in pipelines: 10(GETNEXT), 01(OPEN)
+|
+09:EXCHANGE [HASH(dt.d_year,item.i_brand,item.i_brand_id)]
+| mem-estimate=16.58MB mem-reservation=0B thread-reservation=0
+| tuple-ids=3 row-size=52B cardinality=9.00M cost=3157687
+| in pipelines: 01(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
+Per-Instance Resources: mem-estimate=38.39MB mem-reservation=21.00MB thread-reservation=1
+max-parallelism=160 segment-costs=[1553951857, 34850578] cpu-comparison-result=120 [max(120 (self) vs 25 (sum children))]
+15:TUPLE CACHE
+| cache key: 602237300412b0fe46a455a4c406b9f8
+| input scan node ids: 1
+| estimated serialized size: 481.77MB
+| estimated serialized size per node: 48.18MB
+| cumulative processing cost: 1554138033
+| cache read processing cost: 1195493
+| cache write processing cost: 1363976
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=52B cardinality=9.00M cost=0
+| in pipelines: 01(GETNEXT)
+|
+05:AGGREGATE [STREAMING]
+| output: sum(ss_sales_price)
+| group by: dt.d_year, item.i_brand, item.i_brand_id
+| mem-estimate=10.00MB mem-reservation=5.00MB spill-buffer=256.00KB thread-reservation=0
+| tuple-ids=3 row-size=52B cardinality=9.00M cost=53096454
+| in pipelines: 01(GETNEXT)
+|
+04:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=00
+| hash predicates: store_sales.ss_sold_date_sk = dt.d_date_sk
+| fk/pk conjuncts: store_sales.ss_sold_date_sk = dt.d_date_sk
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1,2,0 row-size=64B cardinality=9.00M cost=3937301
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+|--F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=24.34MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[14167]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: dt.d_date_sk
+| | runtime filters: RF000[bloom] <- dt.d_date_sk
+| | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=6087
+| |
+| 08:EXCHANGE [BROADCAST]
+| | mem-estimate=87.33KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=0 row-size=12B cardinality=6.09K cost=8080
+| | in pipelines: 00(GETNEXT)
+| |
+| F02:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=20.06MB mem-reservation=4.50MB thread-reservation=1
+| max-parallelism=1 segment-costs=[15003]
+| 14:TUPLE CACHE
+| | cache key: c4355c97cf54825ec0d5f5fe4a27616e
+| | input scan node ids: 0
+| | estimated serialized size: 95.11KB
+| | estimated serialized size per node: 95.11KB
+| | cumulative processing cost: 14675
+| | cache read processing cost: 808
+| | cache write processing cost: 262
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=0 row-size=12B cardinality=6.09K cost=0
+| | in pipelines: 00(GETNEXT)
+| |
+| 00:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim dt, RANDOM]
+| HDFS partitions=1/1 files=1 size=2.17MB
+| deterministic scan range assignment: true
+| predicates: dt.d_moy = CAST(11 AS INT)
+| stored statistics:
+| table: rows=73.05K size=2.17MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet statistics predicates: dt.d_moy = CAST(11 AS INT)
+| parquet dictionary predicates: dt.d_moy = CAST(11 AS INT)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=0 row-size=12B cardinality=6.09K cost=14675
+| in pipelines: 00(GETNEXT)
+|
+03:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=01
+| hash predicates: store_sales.ss_item_sk = item.i_item_sk
+| fk/pk conjuncts: store_sales.ss_item_sk = item.i_item_sk
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1,2 row-size=52B cardinality=9.00M cost=3937301
+| in pipelines: 01(GETNEXT), 02(OPEN)
+|
+|--F06:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=24.33MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[865]
+| JOIN BUILD
+| | join-table-id=01 plan-id=02 cohort-id=01
+| | build expressions: item.i_item_sk
+| | runtime filters: RF002[bloom] <- item.i_item_sk, RF003[min_max] <- item.i_item_sk
+| | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=375
+| |
+| 07:EXCHANGE [BROADCAST]
+| | mem-estimate=79.39KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=2 row-size=40B cardinality=375 cost=490
+| | in pipelines: 02(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+| Per-Instance Resources: mem-estimate=20.17MB mem-reservation=4.50MB thread-reservation=1
+| max-parallelism=4 segment-costs=[156141]
+| 13:TUPLE CACHE
+| | cache key: 9667419a8188dd9f26a93dc52053603b
+| | input scan node ids: 2
+| | estimated serialized size: 16.17KB
+| | estimated serialized size per node: 4.04KB
+| | cumulative processing cost: 156093
+| | cache read processing cost: 49
+| | cache write processing cost: 44
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=2 row-size=40B cardinality=375 cost=0
+| | in pipelines: 02(GETNEXT)
+| |
+| 02:SCAN HDFS [tpcds_partitioned_parquet_snap.item, RANDOM]
+| HDFS partitions=1/1 files=1 size=33.54MB
+| deterministic scan range assignment: true
+| predicates: item.i_manufact_id = CAST(816 AS INT)
+| stored statistics:
+| table: rows=360.00K size=33.54MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=101.54K
+| parquet statistics predicates: item.i_manufact_id = CAST(816 AS INT)
+| parquet dictionary predicates: item.i_manufact_id = CAST(816 AS INT)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=2 row-size=40B cardinality=375 cost=156093
+| in pipelines: 02(GETNEXT)
+|
+12:TUPLE CACHE
+| cache key: 5031b6c7488d1b98c1a053d3f3594410
+| input scan node ids: 1
+| estimated serialized size: 137.26MB
+| estimated serialized size per node: 13.73MB
+| cumulative processing cost: 1492980801
+| cache read processing cost: 1195493
+| cache write processing cost: 388602
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=12B cardinality=9.00M(filtered from 8.64G) cost=0
+| in pipelines: 01(GETNEXT)
+|
+01:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ deterministic scan range assignment: true
+ runtime filters: RF003[min_max] -> store_sales.ss_item_sk, RF000[bloom] -> store_sales.ss_sold_date_sk, RF002[bloom] -> store_sales.ss_item_sk
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=1 row-size=12B cardinality=9.00M(filtered from 8.64G) cost=1492980801
+ in pipelines: 01(GETNEXT)
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=383.50MB Threads=19
+Per-Host Resource Estimates: Memory=657MB
+F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=4.11MB mem-reservation=4.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[435] cpu-comparison-result=120 [max(1 (self) vs 120 (sum children))]
+PLAN-ROOT SINK
+| output exprs: dt.d_year, item.i_brand_id, item.i_brand, sum(ss_sales_price)
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=400
+|
+11:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: dt.d_year ASC, sum(ss_sales_price) DESC, item.i_brand_id ASC
+| limit: 100
+| mem-estimate=110.20KB mem-reservation=0B thread-reservation=0
+| tuple-ids=4 row-size=52B cardinality=100 cost=35
+| in pipelines: 06(GETNEXT)
+|
+F03:PLAN FRAGMENT [HASH(dt.d_year,item.i_brand,item.i_brand_id)] hosts=10 instances=20 (adjusted from 120)
+Per-Instance Resources: mem-estimate=50.58MB mem-reservation=34.00MB thread-reservation=1
+max-parallelism=20 segment-costs=[56254141, 117158429, 168] cpu-comparison-result=120 [max(20 (self) vs 120 (sum children))]
+06:TOP-N [LIMIT=100]
+| order by: dt.d_year ASC, sum(ss_sales_price) DESC, item.i_brand_id ASC
+| mem-estimate=5.09KB mem-reservation=0B thread-reservation=0
+| tuple-ids=4 row-size=52B cardinality=100 cost=117158429
+| in pipelines: 06(GETNEXT), 10(OPEN)
+|
+10:AGGREGATE [FINALIZE]
+| output: sum:merge(ss_sales_price)
+| group by: dt.d_year, item.i_brand, item.i_brand_id
+| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=52B cardinality=9.00M cost=53096454
+| in pipelines: 10(GETNEXT), 01(OPEN)
+|
+09:EXCHANGE [HASH(dt.d_year,item.i_brand,item.i_brand_id)]
+| mem-estimate=16.58MB mem-reservation=0B thread-reservation=0
+| tuple-ids=3 row-size=52B cardinality=9.00M cost=3157687
+| in pipelines: 01(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
+Per-Instance Resources: mem-estimate=38.39MB mem-reservation=21.00MB thread-reservation=1
+max-parallelism=160 segment-costs=[1553951857, 34850578] cpu-comparison-result=120 [max(120 (self) vs 25 (sum children))]
+15:TUPLE CACHE
+| cache key: 602237300412b0fe46a455a4c406b9f8
+| input scan node ids: 1
+| estimated serialized size: 481.77MB
+| estimated serialized size per node: 48.18MB
+| cumulative processing cost: 1554138033
+| cache read processing cost: 1195493
+| cache write processing cost: 1363976
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=52B cardinality=9.00M cost=0
+| in pipelines: 01(GETNEXT)
+|
+05:AGGREGATE [STREAMING]
+| output: sum(ss_sales_price)
+| group by: dt.d_year, item.i_brand, item.i_brand_id
+| mem-estimate=10.00MB mem-reservation=5.00MB spill-buffer=256.00KB thread-reservation=0
+| tuple-ids=3 row-size=52B cardinality=9.00M cost=53096454
+| in pipelines: 01(GETNEXT)
+|
+04:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=00
+| hash predicates: store_sales.ss_sold_date_sk = dt.d_date_sk
+| fk/pk conjuncts: store_sales.ss_sold_date_sk = dt.d_date_sk
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1,2,0 row-size=64B cardinality=9.00M cost=3937301
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+|--F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=24.34MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[14167]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: dt.d_date_sk
+| | runtime filters: RF000[bloom] <- dt.d_date_sk
+| | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=6087
+| |
+| 08:EXCHANGE [BROADCAST]
+| | mem-estimate=87.33KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=0 row-size=12B cardinality=6.09K cost=8080
+| | in pipelines: 00(GETNEXT)
+| |
+| F02:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=20.06MB mem-reservation=4.50MB thread-reservation=1
+| max-parallelism=1 segment-costs=[15003]
+| 14:TUPLE CACHE
+| | cache key: c4355c97cf54825ec0d5f5fe4a27616e
+| | input scan node ids: 0
+| | estimated serialized size: 95.11KB
+| | estimated serialized size per node: 95.11KB
+| | cumulative processing cost: 14675
+| | cache read processing cost: 808
+| | cache write processing cost: 262
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=0 row-size=12B cardinality=6.09K cost=0
+| | in pipelines: 00(GETNEXT)
+| |
+| 00:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim dt, RANDOM]
+| HDFS partitions=1/1 files=1 size=2.17MB
+| deterministic scan range assignment: true
+| predicates: dt.d_moy = CAST(11 AS INT)
+| stored statistics:
+| table: rows=73.05K size=2.17MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet statistics predicates: dt.d_moy = CAST(11 AS INT)
+| parquet dictionary predicates: dt.d_moy = CAST(11 AS INT)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=0 row-size=12B cardinality=6.09K cost=14675
+| in pipelines: 00(GETNEXT)
+|
+03:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=01
+| hash predicates: store_sales.ss_item_sk = item.i_item_sk
+| fk/pk conjuncts: store_sales.ss_item_sk = item.i_item_sk
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1,2 row-size=52B cardinality=9.00M cost=3937301
+| in pipelines: 01(GETNEXT), 02(OPEN)
+|
+|--F06:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=24.33MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[865]
+| JOIN BUILD
+| | join-table-id=01 plan-id=02 cohort-id=01
+| | build expressions: item.i_item_sk
+| | runtime filters: RF002[bloom] <- item.i_item_sk, RF003[min_max] <- item.i_item_sk
+| | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=375
+| |
+| 07:EXCHANGE [BROADCAST]
+| | mem-estimate=79.39KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=2 row-size=40B cardinality=375 cost=490
+| | in pipelines: 02(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+| Per-Instance Resources: mem-estimate=20.17MB mem-reservation=4.50MB thread-reservation=1
+| max-parallelism=4 segment-costs=[156141]
+| 13:TUPLE CACHE
+| | cache key: 9667419a8188dd9f26a93dc52053603b
+| | input scan node ids: 2
+| | estimated serialized size: 16.17KB
+| | estimated serialized size per node: 4.04KB
+| | cumulative processing cost: 156093
+| | cache read processing cost: 49
+| | cache write processing cost: 44
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=2 row-size=40B cardinality=375 cost=0
+| | in pipelines: 02(GETNEXT)
+| |
+| 02:SCAN HDFS [tpcds_partitioned_parquet_snap.item, RANDOM]
+| HDFS partitions=1/1 files=1 size=33.54MB
+| deterministic scan range assignment: true
+| predicates: item.i_manufact_id = CAST(816 AS INT)
+| stored statistics:
+| table: rows=360.00K size=33.54MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=101.54K
+| parquet statistics predicates: item.i_manufact_id = CAST(816 AS INT)
+| parquet dictionary predicates: item.i_manufact_id = CAST(816 AS INT)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=2 row-size=40B cardinality=375 cost=156093
+| in pipelines: 02(GETNEXT)
+|
+12:TUPLE CACHE
+| cache key: 5031b6c7488d1b98c1a053d3f3594410
+| input scan node ids: 1
+| estimated serialized size: 137.26MB
+| estimated serialized size per node: 13.73MB
+| cumulative processing cost: 1492980801
+| cache read processing cost: 1195493
+| cache write processing cost: 388602
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=12B cardinality=9.00M(filtered from 8.64G) cost=0
+| in pipelines: 01(GETNEXT)
+|
+01:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ deterministic scan range assignment: true
+ runtime filters: RF003[min_max] -> store_sales.ss_item_sk, RF000[bloom] -> store_sales.ss_sold_date_sk, RF002[bloom] -> store_sales.ss_item_sk
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=1 row-size=12B cardinality=9.00M(filtered from 8.64G) cost=1492980801
+ in pipelines: 01(GETNEXT)
+====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q04.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q04.test
new file mode 100644
index 000000000..cb5efdd2b
--- /dev/null
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q04.test
@@ -0,0 +1,2580 @@
+# TPCDS-Q04
+# start query 4 in stream 0 using template query4.tpl using seed 466720613
+with year_total as (
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,c_preferred_cust_flag customer_preferred_cust_flag
+ ,c_birth_country customer_birth_country
+ ,c_login customer_login
+ ,c_email_address customer_email_address
+ ,d_year dyear
+ ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total
+ ,'s' sale_type
+ from customer
+ ,store_sales
+ ,date_dim
+ where c_customer_sk = ss_customer_sk
+ and ss_sold_date_sk = d_date_sk
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,c_preferred_cust_flag
+ ,c_birth_country
+ ,c_login
+ ,c_email_address
+ ,d_year
+ union all
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,c_preferred_cust_flag customer_preferred_cust_flag
+ ,c_birth_country customer_birth_country
+ ,c_login customer_login
+ ,c_email_address customer_email_address
+ ,d_year dyear
+ ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total
+ ,'c' sale_type
+ from customer
+ ,catalog_sales
+ ,date_dim
+ where c_customer_sk = cs_bill_customer_sk
+ and cs_sold_date_sk = d_date_sk
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,c_preferred_cust_flag
+ ,c_birth_country
+ ,c_login
+ ,c_email_address
+ ,d_year
+union all
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,c_preferred_cust_flag customer_preferred_cust_flag
+ ,c_birth_country customer_birth_country
+ ,c_login customer_login
+ ,c_email_address customer_email_address
+ ,d_year dyear
+ ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total
+ ,'w' sale_type
+ from customer
+ ,web_sales
+ ,date_dim
+ where c_customer_sk = ws_bill_customer_sk
+ and ws_sold_date_sk = d_date_sk
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,c_preferred_cust_flag
+ ,c_birth_country
+ ,c_login
+ ,c_email_address
+ ,d_year
+ )
+ select
+ t_s_secyear.customer_id
+ ,t_s_secyear.customer_first_name
+ ,t_s_secyear.customer_last_name
+ ,t_s_secyear.customer_birth_country
+ from year_total t_s_firstyear
+ ,year_total t_s_secyear
+ ,year_total t_c_firstyear
+ ,year_total t_c_secyear
+ ,year_total t_w_firstyear
+ ,year_total t_w_secyear
+ where t_s_secyear.customer_id = t_s_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_c_secyear.customer_id
+ and t_s_firstyear.customer_id = t_c_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_w_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_w_secyear.customer_id
+ and t_s_firstyear.sale_type = 's'
+ and t_c_firstyear.sale_type = 'c'
+ and t_w_firstyear.sale_type = 'w'
+ and t_s_secyear.sale_type = 's'
+ and t_c_secyear.sale_type = 'c'
+ and t_w_secyear.sale_type = 'w'
+ and t_s_firstyear.dyear = 1999
+ and t_s_secyear.dyear = 1999+1
+ and t_c_firstyear.dyear = 1999
+ and t_c_secyear.dyear = 1999+1
+ and t_w_firstyear.dyear = 1999
+ and t_w_secyear.dyear = 1999+1
+ and t_s_firstyear.year_total > 0
+ and t_c_firstyear.year_total > 0
+ and t_w_firstyear.year_total > 0
+ and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end
+ > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end
+ and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end
+ > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end
+ order by t_s_secyear.customer_id
+ ,t_s_secyear.customer_first_name
+ ,t_s_secyear.customer_last_name
+ ,t_s_secyear.customer_birth_country
+limit 100;
+
+# end query 4 in stream 0 using template query4.tpl
+---- PLAN
+Max Per-Host Resource Reservation: Memory=677.88MB Threads=1
+Per-Host Resource Estimates: Memory=83.62GB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=83.62GB mem-reservation=677.88MB thread-reservation=1 runtime-filters-memory=70.00MB
+| max-parallelism=1 segment-costs=[11764350338, 4555427905, 2367304320, 1287212638, 2992083199, 5889891799, 10459077131, 400]
+PLAN-ROOT SINK
+| output exprs: customer_id, customer_first_name, customer_last_name, customer_birth_country
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=400
+|
+47:TOP-N [LIMIT=100]
+| order by: customer_id ASC, customer_first_name ASC, customer_last_name ASC, customer_birth_country ASC
+| mem-estimate=8.27KB mem-reservation=0B thread-reservation=0
+| tuple-ids=84 row-size=85B cardinality=100 cost=9722546971
+| in pipelines: 47(GETNEXT), 06(OPEN)
+|
+46:HASH JOIN [INNER JOIN]
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| other predicates: CASE WHEN year_total > CAST(0 AS DECIMAL(3,0)) THEN year_total / year_total ELSE NULL END > CASE WHEN year_total > CAST(0 AS DECIMAL(3,0)) THEN year_total / year_total ELSE NULL END, CASE WHEN year_total > CAST(0 AS DECIMAL(3,0)) THEN year_total / year_total ELSE NULL END > CASE WHEN year_total > CAST(0 AS DECIMAL(3,0)) THEN year_total / year_total ELSE NULL END
+| runtime filters: RF001[min_max] <- customer_id
+| mem-estimate=5.57GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26,54,82,68,40 row-size=321B cardinality=591.78M cost=244384131
+| in pipelines: 06(GETNEXT), 20(OPEN)
+|
+|--14:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=40 row-size=44B cardinality=80.79M cost=13344408
+| | in pipelines: 20(GETNEXT)
+| |
+| 20:AGGREGATE [FINALIZE]
+| | output: sum((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / CAST(2 AS DECIMAL(3,0))))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | having: sum((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / 2)) > CAST(0 AS DECIMAL(3,0))
+| | mem-estimate=6.09GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=35 row-size=169B cardinality=80.79M cost=4768537611
+| | in pipelines: 20(GETNEXT), 16(OPEN)
+| |
+| 19:HASH JOIN [INNER JOIN]
+| | hash predicates: cs_bill_customer_sk = c_customer_sk
+| | fk/pk conjuncts: cs_bill_customer_sk = c_customer_sk
+| | runtime filters: RF030[bloom] <- c_customer_sk, RF031[min_max] <- c_customer_sk
+| | mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=33,34,32 row-size=185B cardinality=807.87M cost=402121453
+| | in pipelines: 16(GETNEXT), 15(OPEN)
+| |
+| |--15:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=32 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 15(GETNEXT)
+| |
+| 18:HASH JOIN [INNER JOIN]
+| | hash predicates: cs_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: cs_sold_date_sk = d_date_sk
+| | runtime filters: RF032[bloom] <- d_date_sk
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=33,34 row-size=32B cardinality=880.06M cost=385202450
+| | in pipelines: 16(GETNEXT), 17(OPEN)
+| |
+| |--53:TUPLE CACHE
+| | | cache key: 09df1e7b5f44dfdb1fd5f1f4c9cf3018
+| | | input scan node ids: 17
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=34 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 17(GETNEXT)
+| | |
+| | 17:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=34 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 17(GETNEXT)
+| |
+| 16:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales]
+| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| runtime filters: RF031[min_max] -> cs_bill_customer_sk, RF032[bloom] -> cs_sold_date_sk, RF030[bloom] -> cs_bill_customer_sk
+| stored statistics:
+| table: rows=4.32G size=280.96GB
+| partitions: 1831/1831 rows=4.32G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=21.52M est-scan-range=374(filtered from 1831)
+| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| tuple-ids=33 row-size=24B cardinality=880.06M(filtered from 4.32G) cost=304148590
+| in pipelines: 16(GETNEXT)
+|
+45:HASH JOIN [INNER JOIN]
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| runtime filters: RF003[min_max] <- customer_id
+| mem-estimate=2.57GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26,54,82,68 row-size=277B cardinality=219.75M cost=122068838
+| in pipelines: 06(GETNEXT), 34(OPEN)
+|
+|--28:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=68 row-size=44B cardinality=40.65M cost=6714166
+| | in pipelines: 34(GETNEXT)
+| |
+| 34:AGGREGATE [FINALIZE]
+| | output: sum((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / CAST(2 AS DECIMAL(3,0))))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | having: sum((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / 2)) > CAST(0 AS DECIMAL(3,0))
+| | mem-estimate=5.56GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=67 row-size=169B cardinality=40.65M cost=2399263927
+| | in pipelines: 34(GETNEXT), 30(OPEN)
+| |
+| 33:HASH JOIN [INNER JOIN]
+| | hash predicates: ws_bill_customer_sk = c_customer_sk
+| | fk/pk conjuncts: ws_bill_customer_sk = c_customer_sk
+| | runtime filters: RF027[min_max] <- c_customer_sk
+| | mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=65,66,64 row-size=185B cardinality=406.48M cost=216950480
+| | in pipelines: 30(GETNEXT), 29(OPEN)
+| |
+| |--29:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=64 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 29(GETNEXT)
+| |
+| 32:HASH JOIN [INNER JOIN]
+| | hash predicates: ws_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: ws_sold_date_sk = d_date_sk
+| | runtime filters: RF028[bloom] <- d_date_sk
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=65,66 row-size=32B cardinality=441.70M cost=193334203
+| | in pipelines: 30(GETNEXT), 31(OPEN)
+| |
+| |--52:TUPLE CACHE
+| | | cache key: 09df1e7b5f44dfdb1fd5f1f4c9cf3018
+| | | input scan node ids: 31
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=66 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 31(GETNEXT)
+| | |
+| | 31:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=66 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 31(GETNEXT)
+| |
+| 30:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
+| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| runtime filters: RF027[min_max] -> ws_bill_customer_sk, RF028[bloom] -> ws_sold_date_sk
+| stored statistics:
+| table: rows=2.16G size=145.75GB
+| partitions: 1824/1824 rows=2.16G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=374(filtered from 1824)
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=65 row-size=24B cardinality=441.70M(filtered from 2.16G) cost=152652894
+| in pipelines: 30(GETNEXT)
+|
+44:HASH JOIN [INNER JOIN]
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| runtime filters: RF005[min_max] <- customer_id
+| mem-estimate=1.98GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26,54,82 row-size=233B cardinality=162.19M cost=100990594
+| in pipelines: 06(GETNEXT), 41(OPEN)
+|
+|--35:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=82 row-size=44B cardinality=30.00M cost=4955400
+| | in pipelines: 41(GETNEXT)
+| |
+| 41:AGGREGATE [FINALIZE]
+| | output: sum((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / CAST(2 AS DECIMAL(3,0))))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=5.56GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=81 row-size=169B cardinality=30.00M cost=694393366
+| | in pipelines: 41(GETNEXT), 37(OPEN)
+| |
+| 40:HASH JOIN [INNER JOIN]
+| | hash predicates: ws_bill_customer_sk = c_customer_sk
+| | fk/pk conjuncts: ws_bill_customer_sk = c_customer_sk
+| | runtime filters: RF023[min_max] <- c_customer_sk
+| | mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=79,80,78 row-size=185B cardinality=406.48M cost=216950480
+| | in pipelines: 37(GETNEXT), 36(OPEN)
+| |
+| |--36:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | runtime filters: RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=78 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 36(GETNEXT)
+| |
+| 39:HASH JOIN [INNER JOIN]
+| | hash predicates: ws_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: ws_sold_date_sk = d_date_sk
+| | runtime filters: RF024[bloom] <- d_date_sk
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=79,80 row-size=32B cardinality=441.70M cost=193334203
+| | in pipelines: 37(GETNEXT), 38(OPEN)
+| |
+| |--51:TUPLE CACHE
+| | | cache key: 7b1386755ea9bf5bc7f77efa229d8f45
+| | | input scan node ids: 38
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=80 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 38(GETNEXT)
+| | |
+| | 38:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=80 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 38(GETNEXT)
+| |
+| 37:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
+| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| runtime filters: RF023[min_max] -> ws_bill_customer_sk, RF024[bloom] -> ws_sold_date_sk
+| stored statistics:
+| table: rows=2.16G size=145.75GB
+| partitions: 1824/1824 rows=2.16G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=374(filtered from 1824)
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=79 row-size=24B cardinality=441.70M(filtered from 2.16G) cost=152652894
+| in pipelines: 37(GETNEXT)
+|
+43:HASH JOIN [INNER JOIN]
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| runtime filters: RF007[min_max] <- customer_id
+| mem-estimate=1.98GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26,54 row-size=189B cardinality=162.19M cost=100990594
+| in pipelines: 06(GETNEXT), 27(OPEN)
+|
+|--21:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=54 row-size=44B cardinality=30.00M cost=4955400
+| | in pipelines: 27(GETNEXT)
+| |
+| 27:AGGREGATE [FINALIZE]
+| | output: sum((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / CAST(2 AS DECIMAL(3,0))))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=6.09GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=49 row-size=169B cardinality=30.00M cost=1245950132
+| | in pipelines: 27(GETNEXT), 23(OPEN)
+| |
+| 26:HASH JOIN [INNER JOIN]
+| | hash predicates: cs_bill_customer_sk = c_customer_sk
+| | fk/pk conjuncts: cs_bill_customer_sk = c_customer_sk
+| | runtime filters: RF018[bloom] <- c_customer_sk, RF019[min_max] <- c_customer_sk
+| | mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=47,48,46 row-size=185B cardinality=807.87M cost=402121453
+| | in pipelines: 23(GETNEXT), 22(OPEN)
+| |
+| |--22:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=46 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 22(GETNEXT)
+| |
+| 25:HASH JOIN [INNER JOIN]
+| | hash predicates: cs_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: cs_sold_date_sk = d_date_sk
+| | runtime filters: RF020[bloom] <- d_date_sk
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=47,48 row-size=32B cardinality=880.06M cost=385202450
+| | in pipelines: 23(GETNEXT), 24(OPEN)
+| |
+| |--50:TUPLE CACHE
+| | | cache key: 7b1386755ea9bf5bc7f77efa229d8f45
+| | | input scan node ids: 24
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=48 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 24(GETNEXT)
+| | |
+| | 24:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=48 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 24(GETNEXT)
+| |
+| 23:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales]
+| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| runtime filters: RF019[min_max] -> cs_bill_customer_sk, RF020[bloom] -> cs_sold_date_sk, RF018[bloom] -> cs_bill_customer_sk
+| stored statistics:
+| table: rows=4.32G size=280.96GB
+| partitions: 1831/1831 rows=4.32G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=21.52M est-scan-range=374(filtered from 1831)
+| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| tuple-ids=47 row-size=24B cardinality=880.06M(filtered from 4.32G) cost=304148590
+| in pipelines: 23(GETNEXT)
+|
+42:HASH JOIN [INNER JOIN]
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| runtime filters: RF009[min_max] <- customer_id
+| mem-estimate=3.56GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26 row-size=145B cardinality=162.19M cost=100990594
+| in pipelines: 06(GETNEXT), 13(OPEN)
+|
+|--07:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=26 row-size=101B cardinality=30.00M cost=10345479
+| | in pipelines: 13(GETNEXT)
+| |
+| 13:AGGREGATE [FINALIZE]
+| | output: sum(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / CAST(2 AS DECIMAL(3,0)))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=7.16GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=17 row-size=169B cardinality=30.00M cost=2364508786
+| | in pipelines: 13(GETNEXT), 09(OPEN)
+| |
+| 12:HASH JOIN [INNER JOIN]
+| | hash predicates: ss_customer_sk = c_customer_sk
+| | fk/pk conjuncts: ss_customer_sk = c_customer_sk
+| | runtime filters: RF014[bloom] <- c_customer_sk, RF015[min_max] <- c_customer_sk
+| | mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=15,16,14 row-size=185B cardinality=1.62G cost=777080012
+| | in pipelines: 09(GETNEXT), 08(OPEN)
+| |
+| |--08:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF007[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=14 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 08(GETNEXT)
+| |
+| 11:HASH JOIN [INNER JOIN]
+| | hash predicates: ss_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: ss_sold_date_sk = d_date_sk
+| | runtime filters: RF016[bloom] <- d_date_sk
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=15,16 row-size=32B cardinality=1.77G cost=773341361
+| | in pipelines: 09(GETNEXT), 10(OPEN)
+| |
+| |--49:TUPLE CACHE
+| | | cache key: 7b1386755ea9bf5bc7f77efa229d8f45
+| | | input scan node ids: 10
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=16 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 10(GETNEXT)
+| | |
+| | 10:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=16 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 10(GETNEXT)
+| |
+| 09:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales]
+| HDFS partitions=1824/1824 files=1824 size=389.90GB
+| runtime filters: RF015[min_max] -> ss_customer_sk, RF016[bloom] -> ss_sold_date_sk, RF014[bloom] -> ss_customer_sk
+| stored statistics:
+| table: rows=8.64G size=389.90GB
+| partitions: 1824/1824 rows=8.64G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
+| mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+| tuple-ids=15 row-size=24B cardinality=1.77G(filtered from 8.64G) cost=610616051
+| in pipelines: 09(GETNEXT)
+|
+00:UNION
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=12 row-size=44B cardinality=162.19M cost=26790556
+| in pipelines: 06(GETNEXT)
+|
+06:AGGREGATE [FINALIZE]
+| output: sum(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / CAST(2 AS DECIMAL(3,0)))
+| group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| having: sum(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / 2) > CAST(0 AS DECIMAL(3,0))
+| mem-estimate=7.16GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=169B cardinality=162.19M cost=9573431219
+| in pipelines: 06(GETNEXT), 02(OPEN)
+|
+05:HASH JOIN [INNER JOIN]
+| hash predicates: ss_customer_sk = c_customer_sk
+| fk/pk conjuncts: ss_customer_sk = c_customer_sk
+| runtime filters: RF010[bloom] <- c_customer_sk, RF011[min_max] <- c_customer_sk
+| mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1,2,0 row-size=185B cardinality=1.62G cost=777080012
+| in pipelines: 02(GETNEXT), 01(OPEN)
+|
+|--01:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
+| HDFS partitions=1/1 files=1 size=1.55GB
+| runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF007[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF009[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| stored statistics:
+| table: rows=30.00M size=1.55GB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.10M
+| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| tuple-ids=0 row-size=153B cardinality=30.00M cost=29871228
+| in pipelines: 01(GETNEXT)
+|
+04:HASH JOIN [INNER JOIN]
+| hash predicates: ss_sold_date_sk = d_date_sk
+| fk/pk conjuncts: ss_sold_date_sk = d_date_sk
+| runtime filters: RF012[bloom] <- d_date_sk
+| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1,2 row-size=32B cardinality=1.77G cost=773341361
+| in pipelines: 02(GETNEXT), 03(OPEN)
+|
+|--48:TUPLE CACHE
+| | cache key: 09df1e7b5f44dfdb1fd5f1f4c9cf3018
+| | input scan node ids: 3
+| | estimated serialized size: 4.37KB
+| | estimated serialized size per node: 4.37KB
+| | cumulative processing cost: 10467
+| | cache read processing cost: 49
+| | cache write processing cost: 12
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=2 row-size=8B cardinality=373 cost=0
+| | in pipelines: 03(GETNEXT)
+| |
+| 03:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| HDFS partitions=1/1 files=1 size=2.17MB
+| deterministic scan range assignment: true
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| stored statistics:
+| table: rows=73.05K size=2.17MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=2 row-size=8B cardinality=373 cost=10467
+| in pipelines: 03(GETNEXT)
+|
+02:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ runtime filters: RF011[min_max] -> ss_customer_sk, RF012[bloom] -> ss_sold_date_sk, RF010[bloom] -> ss_customer_sk
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=1 row-size=24B cardinality=1.77G(filtered from 8.64G) cost=610616051
+ in pipelines: 02(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=10.37GB Threads=379
+Per-Host Resource Estimates: Memory=182.31GB
+F37:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=5.02MB mem-reservation=4.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[442] cpu-comparison-result=1440 [max(1 (self) vs 1440 (sum children))]
+PLAN-ROOT SINK
+| output exprs: customer_id, customer_first_name, customer_last_name, customer_birth_country
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=400
+|
+84:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: customer_id ASC, customer_first_name ASC, customer_last_name ASC, customer_birth_country ASC
+| limit: 100
+| mem-estimate=1.02MB mem-reservation=0B thread-reservation=0
+| tuple-ids=84 row-size=85B cardinality=100 cost=42
+| in pipelines: 47(GETNEXT)
+|
+F12:PLAN FRAGMENT [HASH(customer_id)] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=15.63MB mem-reservation=0B thread-reservation=1
+max-parallelism=1030 segment-costs=[10231383661, 253] cpu-comparison-result=1440 [max(120 (self) vs 1440 (sum children))]
+47:TOP-N [LIMIT=100]
+| order by: customer_id ASC, customer_first_name ASC, customer_last_name ASC, customer_birth_country ASC
+| mem-estimate=8.27KB mem-reservation=0B thread-reservation=0
+| tuple-ids=84 row-size=85B cardinality=100 cost=9722546971
+| in pipelines: 47(GETNEXT), 52(OPEN)
+|
+46:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=00
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| other predicates: CASE WHEN year_total > CAST(0 AS DECIMAL(3,0)) THEN year_total / year_total ELSE NULL END > CASE WHEN year_total > CAST(0 AS DECIMAL(3,0)) THEN year_total / year_total ELSE NULL END, CASE WHEN year_total > CAST(0 AS DECIMAL(3,0)) THEN year_total / year_total ELSE NULL END > CASE WHEN year_total > CAST(0 AS DECIMAL(3,0)) THEN year_total / year_total ELSE NULL END
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26,54,82,68,40 row-size=321B cardinality=591.78M cost=163597061
+| in pipelines: 52(GETNEXT), 82(OPEN)
+|
+|--F38:PLAN FRAGMENT [HASH(customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=63.13MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=1030 segment-costs=[106113816] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: customer_id
+| | runtime filters: RF001[min_max] <- customer_id
+| | mem-estimate=47.51MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=80787070
+| |
+| 83:EXCHANGE [HASH(customer_id)]
+| | mem-estimate=15.62MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=40 row-size=44B cardinality=80.79M cost=25326746
+| | in pipelines: 82(GETNEXT)
+| |
+| F36:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.19GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[5486874642, 283884148] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 14:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=40 row-size=44B cardinality=80.79M cost=13344408
+| | in pipelines: 82(GETNEXT)
+| |
+| 82:AGGREGATE [FINALIZE]
+| | output: sum:merge((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / 2))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | having: sum((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / 2)) > CAST(0 AS DECIMAL(3,0))
+| | mem-estimate=1.14GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=35 row-size=169B cardinality=80.79M cost=4768537611
+| | in pipelines: 82(GETNEXT), 16(OPEN)
+| |
+| 81:EXCHANGE [HASH(c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year)]
+| | mem-estimate=30.29MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=35 row-size=169B cardinality=807.87M cost=718337031
+| | in pipelines: 16(GETNEXT)
+| |
+| F34:PLAN FRAGMENT [HASH(cs_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=968.12MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=930 segment-costs=[5400364645, 9216371342] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 20:AGGREGATE [STREAMING]
+| | output: sum((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / CAST(2 AS DECIMAL(3,0))))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=872.27MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=35 row-size=169B cardinality=807.87M cost=4768537611
+| | in pipelines: 16(GETNEXT)
+| |
+| 19:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=01
+| | hash predicates: cs_bill_customer_sk = c_customer_sk
+| | fk/pk conjuncts: cs_bill_customer_sk = c_customer_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=33,34,32 row-size=185B cardinality=807.87M cost=372121453
+| | in pipelines: 16(GETNEXT), 15(OPEN)
+| |
+| |--F39:PLAN FRAGMENT [HASH(cs_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=70.45MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | | max-parallelism=930 segment-costs=[54467197]
+| | JOIN BUILD
+| | | join-table-id=01 plan-id=02 cohort-id=02
+| | | build expressions: c_customer_sk
+| | | runtime filters: RF030[bloom] <- c_customer_sk, RF031[min_max] <- c_customer_sk
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | |
+| | 80:EXCHANGE [HASH(c_customer_sk)]
+| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=32 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 15(GETNEXT)
+| | |
+| | F33:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[341205999]
+| | 15:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=32 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 15(GETNEXT)
+| |
+| 79:EXCHANGE [HASH(cs_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=33,34 row-size=32B cardinality=880.06M cost=259705581
+| | in pipelines: 16(GETNEXT)
+| |
+| F31:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=17.00MB mem-reservation=17.00MB thread-reservation=0 runtime-filters-memory=17.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=1.00MB thread-reservation=1
+| max-parallelism=350 segment-costs=[3409790834]
+| 18:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=02
+| | hash predicates: cs_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: cs_sold_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=33,34 row-size=32B cardinality=880.06M cost=385202077
+| | in pipelines: 16(GETNEXT), 17(OPEN)
+| |
+| |--F40:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=02 plan-id=03 cohort-id=02
+| | | build expressions: d_date_sk
+| | | runtime filters: RF032[bloom] <- d_date_sk
+| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 78:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=34 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 17(GETNEXT)
+| | |
+| | F32:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 90:TUPLE CACHE
+| | | cache key: 09df1e7b5f44dfdb1fd5f1f4c9cf3018
+| | | input scan node ids: 17
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=34 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 17(GETNEXT)
+| | |
+| | 17:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=34 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 17(GETNEXT)
+| |
+| 16:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
+| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| runtime filters: RF031[min_max] -> cs_bill_customer_sk, RF032[bloom] -> cs_sold_date_sk, RF030[bloom] -> cs_bill_customer_sk
+| stored statistics:
+| table: rows=4.32G size=280.96GB
+| partitions: 1831/1831 rows=4.32G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=21.52M est-scan-range=374(filtered from 1831)
+| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| tuple-ids=33 row-size=24B cardinality=880.06M(filtered from 4.32G) cost=304148590
+| in pipelines: 16(GETNEXT)
+|
+45:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=03
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26,54,82,68 row-size=277B cardinality=219.75M cost=81421260
+| in pipelines: 52(GETNEXT), 76(OPEN)
+|
+|--F41:PLAN FRAGMENT [HASH(customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=49.62MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=1030 segment-costs=[53390593] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| JOIN BUILD
+| | join-table-id=03 plan-id=04 cohort-id=01
+| | build expressions: customer_id
+| | runtime filters: RF003[min_max] <- customer_id
+| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=40647578
+| |
+| 77:EXCHANGE [HASH(customer_id)]
+| | mem-estimate=15.62MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=68 row-size=44B cardinality=40.65M cost=12743015
+| | in pipelines: 76(GETNEXT)
+| |
+| F30:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=637.96MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[2760691322, 142834775] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 28:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=68 row-size=44B cardinality=40.65M cost=6714166
+| | in pipelines: 76(GETNEXT)
+| |
+| 76:AGGREGATE [FINALIZE]
+| | output: sum:merge((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / 2))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | having: sum((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / 2)) > CAST(0 AS DECIMAL(3,0))
+| | mem-estimate=585.17MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=67 row-size=169B cardinality=40.65M cost=2399263927
+| | in pipelines: 76(GETNEXT), 30(OPEN)
+| |
+| 75:EXCHANGE [HASH(c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year)]
+| | mem-estimate=30.29MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=67 row-size=169B cardinality=406.48M cost=361427395
+| | in pipelines: 30(GETNEXT)
+| |
+| F28:PLAN FRAGMENT [HASH(ws_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=534.73MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=470 segment-costs=[2716561250, 4637167430] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 34:AGGREGATE [STREAMING]
+| | output: sum((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / CAST(2 AS DECIMAL(3,0))))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=438.88MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=67 row-size=169B cardinality=406.48M cost=2399263927
+| | in pipelines: 30(GETNEXT)
+| |
+| 33:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=04
+| | hash predicates: ws_bill_customer_sk = c_customer_sk
+| | fk/pk conjuncts: ws_bill_customer_sk = c_customer_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=65,66,64 row-size=185B cardinality=406.48M cost=186950480
+| | in pipelines: 30(GETNEXT), 29(OPEN)
+| |
+| |--F42:PLAN FRAGMENT [HASH(ws_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=54.45MB mem-reservation=34.00MB thread-reservation=1
+| | | max-parallelism=470 segment-costs=[54467197]
+| | JOIN BUILD
+| | | join-table-id=04 plan-id=05 cohort-id=03
+| | | build expressions: c_customer_sk
+| | | runtime filters: RF027[min_max] <- c_customer_sk
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | |
+| | 74:EXCHANGE [HASH(c_customer_sk)]
+| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=64 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 29(GETNEXT)
+| | |
+| | F27:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[341205999]
+| | 29:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=64 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 29(GETNEXT)
+| |
+| 73:EXCHANGE [HASH(ws_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=65,66 row-size=32B cardinality=441.70M cost=130346843
+| | in pipelines: 30(GETNEXT)
+| |
+| F25:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=180 segment-costs=[1711382054]
+| 32:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=05
+| | hash predicates: ws_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: ws_sold_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=65,66 row-size=32B cardinality=441.70M cost=193333830
+| | in pipelines: 30(GETNEXT), 31(OPEN)
+| |
+| |--F43:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=05 plan-id=06 cohort-id=03
+| | | build expressions: d_date_sk
+| | | runtime filters: RF028[bloom] <- d_date_sk
+| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 72:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=66 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 31(GETNEXT)
+| | |
+| | F26:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 89:TUPLE CACHE
+| | | cache key: 09df1e7b5f44dfdb1fd5f1f4c9cf3018
+| | | input scan node ids: 31
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=66 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 31(GETNEXT)
+| | |
+| | 31:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=66 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 31(GETNEXT)
+| |
+| 30:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
+| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| runtime filters: RF027[min_max] -> ws_bill_customer_sk, RF028[bloom] -> ws_sold_date_sk
+| stored statistics:
+| table: rows=2.16G size=145.75GB
+| partitions: 1824/1824 rows=2.16G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=374(filtered from 1824)
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=65 row-size=24B cardinality=441.70M(filtered from 2.16G) cost=152652894
+| in pipelines: 30(GETNEXT)
+|
+44:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=06
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26,54,82 row-size=233B cardinality=162.19M cost=70990594
+| in pipelines: 52(GETNEXT), 70(OPEN)
+|
+|--F44:PLAN FRAGMENT [HASH(customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=49.62MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=1030 segment-costs=[39405000] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| JOIN BUILD
+| | join-table-id=06 plan-id=07 cohort-id=01
+| | build expressions: customer_id
+| | runtime filters: RF005[min_max] <- customer_id
+| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| |
+| 71:EXCHANGE [HASH(customer_id)]
+| | mem-estimate=15.62MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=82 row-size=44B cardinality=30.00M cost=9405000
+| | in pipelines: 70(GETNEXT)
+| |
+| F24:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=606.14MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[1005785180, 105419400] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 35:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=82 row-size=44B cardinality=30.00M cost=4955400
+| | in pipelines: 70(GETNEXT)
+| |
+| 70:AGGREGATE [FINALIZE]
+| | output: sum:merge((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / 2))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=553.34MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=81 row-size=169B cardinality=30.00M cost=664015288
+| | in pipelines: 70(GETNEXT), 37(OPEN)
+| |
+| 69:EXCHANGE [HASH(c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year)]
+| | mem-estimate=30.29MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=81 row-size=169B cardinality=384.37M cost=341769892
+| | in pipelines: 37(GETNEXT)
+| |
+| F22:PLAN FRAGMENT [HASH(ws_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=534.73MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=440 segment-costs=[2616446902, 4384958733] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 41:AGGREGATE [STREAMING]
+| | output: sum((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / CAST(2 AS DECIMAL(3,0))))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=438.88MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=81 row-size=169B cardinality=384.37M cost=2299149579
+| | in pipelines: 37(GETNEXT)
+| |
+| 40:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=07
+| | hash predicates: ws_bill_customer_sk = c_customer_sk
+| | fk/pk conjuncts: ws_bill_customer_sk = c_customer_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=79,80,78 row-size=185B cardinality=406.48M cost=186950480
+| | in pipelines: 37(GETNEXT), 36(OPEN)
+| |
+| |--F45:PLAN FRAGMENT [HASH(ws_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=54.45MB mem-reservation=34.00MB thread-reservation=1
+| | | max-parallelism=440 segment-costs=[54467197]
+| | JOIN BUILD
+| | | join-table-id=07 plan-id=08 cohort-id=04
+| | | build expressions: c_customer_sk
+| | | runtime filters: RF023[min_max] <- c_customer_sk
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | |
+| | 68:EXCHANGE [HASH(c_customer_sk)]
+| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=78 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 36(GETNEXT)
+| | |
+| | F21:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[341205999]
+| | 36:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | runtime filters: RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=78 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 36(GETNEXT)
+| |
+| 67:EXCHANGE [HASH(ws_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=79,80 row-size=32B cardinality=441.70M cost=130346843
+| | in pipelines: 37(GETNEXT)
+| |
+| F19:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=180 segment-costs=[1711382054]
+| 39:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=08
+| | hash predicates: ws_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: ws_sold_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=79,80 row-size=32B cardinality=441.70M cost=193333830
+| | in pipelines: 37(GETNEXT), 38(OPEN)
+| |
+| |--F46:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=08 plan-id=09 cohort-id=04
+| | | build expressions: d_date_sk
+| | | runtime filters: RF024[bloom] <- d_date_sk
+| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 66:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=80 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 38(GETNEXT)
+| | |
+| | F20:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 88:TUPLE CACHE
+| | | cache key: 7b1386755ea9bf5bc7f77efa229d8f45
+| | | input scan node ids: 38
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=80 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 38(GETNEXT)
+| | |
+| | 38:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=80 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 38(GETNEXT)
+| |
+| 37:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
+| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| runtime filters: RF023[min_max] -> ws_bill_customer_sk, RF024[bloom] -> ws_sold_date_sk
+| stored statistics:
+| table: rows=2.16G size=145.75GB
+| partitions: 1824/1824 rows=2.16G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=374(filtered from 1824)
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=79 row-size=24B cardinality=441.70M(filtered from 2.16G) cost=152652894
+| in pipelines: 37(GETNEXT)
+|
+43:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=09
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26,54 row-size=189B cardinality=162.19M cost=70990594
+| in pipelines: 52(GETNEXT), 64(OPEN)
+|
+|--F47:PLAN FRAGMENT [HASH(customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=49.62MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=1030 segment-costs=[39405000] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| JOIN BUILD
+| | join-table-id=09 plan-id=10 cohort-id=01
+| | build expressions: customer_id
+| | runtime filters: RF007[min_max] <- customer_id
+| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| |
+| 65:EXCHANGE [HASH(customer_id)]
+| | mem-estimate=15.62MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=54 row-size=44B cardinality=30.00M cost=9405000
+| | in pipelines: 64(GETNEXT)
+| |
+| F18:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.07GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[1773652148, 105419400] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 21:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=54 row-size=44B cardinality=30.00M cost=4955400
+| | in pipelines: 64(GETNEXT)
+| |
+| 64:AGGREGATE [FINALIZE]
+| | output: sum:merge((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / 2))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=1.02GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=49 row-size=169B cardinality=30.00M cost=1130209988
+| | in pipelines: 64(GETNEXT), 23(OPEN)
+| |
+| 63:EXCHANGE [HASH(c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year)]
+| | mem-estimate=30.29MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=49 row-size=169B cardinality=723.64M cost=643442160
+| | in pipelines: 23(GETNEXT)
+| |
+| F16:PLAN FRAGMENT [HASH(cs_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=968.12MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=830 segment-costs=[5018930073, 8255458964] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 27:AGGREGATE [STREAMING]
+| | output: sum((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / CAST(2 AS DECIMAL(3,0))))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=872.27MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=49 row-size=169B cardinality=723.64M cost=4387103039
+| | in pipelines: 23(GETNEXT)
+| |
+| 26:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=10
+| | hash predicates: cs_bill_customer_sk = c_customer_sk
+| | fk/pk conjuncts: cs_bill_customer_sk = c_customer_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=47,48,46 row-size=185B cardinality=807.87M cost=372121453
+| | in pipelines: 23(GETNEXT), 22(OPEN)
+| |
+| |--F48:PLAN FRAGMENT [HASH(cs_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=70.45MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | | max-parallelism=830 segment-costs=[54467197]
+| | JOIN BUILD
+| | | join-table-id=10 plan-id=11 cohort-id=05
+| | | build expressions: c_customer_sk
+| | | runtime filters: RF018[bloom] <- c_customer_sk, RF019[min_max] <- c_customer_sk
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | |
+| | 62:EXCHANGE [HASH(c_customer_sk)]
+| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=46 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 22(GETNEXT)
+| | |
+| | F15:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[341205999]
+| | 22:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=46 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 22(GETNEXT)
+| |
+| 61:EXCHANGE [HASH(cs_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=47,48 row-size=32B cardinality=880.06M cost=259705581
+| | in pipelines: 23(GETNEXT)
+| |
+| F13:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=17.00MB mem-reservation=17.00MB thread-reservation=0 runtime-filters-memory=17.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=1.00MB thread-reservation=1
+| max-parallelism=350 segment-costs=[3409790834]
+| 25:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=11
+| | hash predicates: cs_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: cs_sold_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=47,48 row-size=32B cardinality=880.06M cost=385202077
+| | in pipelines: 23(GETNEXT), 24(OPEN)
+| |
+| |--F49:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=11 plan-id=12 cohort-id=05
+| | | build expressions: d_date_sk
+| | | runtime filters: RF020[bloom] <- d_date_sk
+| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 60:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=48 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 24(GETNEXT)
+| | |
+| | F14:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 87:TUPLE CACHE
+| | | cache key: 7b1386755ea9bf5bc7f77efa229d8f45
+| | | input scan node ids: 24
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=48 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 24(GETNEXT)
+| | |
+| | 24:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=48 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 24(GETNEXT)
+| |
+| 23:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
+| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| runtime filters: RF019[min_max] -> cs_bill_customer_sk, RF020[bloom] -> cs_sold_date_sk, RF018[bloom] -> cs_bill_customer_sk
+| stored statistics:
+| table: rows=4.32G size=280.96GB
+| partitions: 1831/1831 rows=4.32G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=21.52M est-scan-range=374(filtered from 1831)
+| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| tuple-ids=47 row-size=24B cardinality=880.06M(filtered from 4.32G) cost=304148590
+| in pipelines: 23(GETNEXT)
+|
+42:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=12
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26 row-size=145B cardinality=162.19M cost=70990594
+| in pipelines: 52(GETNEXT), 57(OPEN)
+|
+|--F50:PLAN FRAGMENT [HASH(customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=56.27MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=1030 segment-costs=[47226566] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| JOIN BUILD
+| | join-table-id=12 plan-id=13 cohort-id=01
+| | build expressions: customer_id
+| | runtime filters: RF009[min_max] <- customer_id
+| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| |
+| 59:EXCHANGE [HASH(customer_id)]
+| | mem-estimate=22.27MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=26 row-size=101B cardinality=30.00M cost=17226566
+| | in pipelines: 57(GETNEXT)
+| |
+| F11:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.91GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[3091124881, 220311407] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 07:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=26 row-size=101B cardinality=30.00M cost=10345479
+| | in pipelines: 57(GETNEXT)
+| |
+| 57:AGGREGATE [FINALIZE]
+| | output: sum:merge(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / 2)
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=1.84GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=17 row-size=169B cardinality=30.00M cost=1930086569
+| | in pipelines: 57(GETNEXT), 09(OPEN)
+| |
+| 56:EXCHANGE [HASH(c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year)]
+| | mem-estimate=30.29MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=17 row-size=169B cardinality=1.31G cost=1161038312
+| | in pipelines: 09(GETNEXT)
+| |
+| F09:PLAN FRAGMENT [HASH(ss_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.80GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=1490 segment-costs=[9410215497, 14896294857] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 13:AGGREGATE [STREAMING]
+| | output: sum(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / CAST(2 AS DECIMAL(3,0)))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=1.71GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=17 row-size=169B cardinality=1.31G cost=8141744291
+| | in pipelines: 09(GETNEXT)
+| |
+| 12:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=13
+| | hash predicates: ss_customer_sk = c_customer_sk
+| | fk/pk conjuncts: ss_customer_sk = c_customer_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=15,16,14 row-size=185B cardinality=1.62G cost=747080012
+| | in pipelines: 09(GETNEXT), 08(OPEN)
+| |
+| |--F51:PLAN FRAGMENT [HASH(ss_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=70.45MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | | max-parallelism=1490 segment-costs=[54467197]
+| | JOIN BUILD
+| | | join-table-id=13 plan-id=14 cohort-id=06
+| | | build expressions: c_customer_sk
+| | | runtime filters: RF014[bloom] <- c_customer_sk, RF015[min_max] <- c_customer_sk
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | |
+| | 55:EXCHANGE [HASH(c_customer_sk)]
+| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=14 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 08(GETNEXT)
+| | |
+| | F08:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[341205999]
+| | 08:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF007[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=14 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 08(GETNEXT)
+| |
+| 54:EXCHANGE [HASH(ss_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=15,16 row-size=32B cardinality=1.77G cost=521391194
+| | in pipelines: 09(GETNEXT)
+| |
+| F06:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=17.00MB mem-reservation=17.00MB thread-reservation=0 runtime-filters-memory=17.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=8.00MB thread-reservation=1
+| max-parallelism=374 segment-costs=[6845578389]
+| 11:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=14
+| | hash predicates: ss_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: ss_sold_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=15,16 row-size=32B cardinality=1.77G cost=773340988
+| | in pipelines: 09(GETNEXT), 10(OPEN)
+| |
+| |--F52:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=14 plan-id=15 cohort-id=06
+| | | build expressions: d_date_sk
+| | | runtime filters: RF016[bloom] <- d_date_sk
+| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 53:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=16 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 10(GETNEXT)
+| | |
+| | F07:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 86:TUPLE CACHE
+| | | cache key: 7b1386755ea9bf5bc7f77efa229d8f45
+| | | input scan node ids: 10
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=16 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 10(GETNEXT)
+| | |
+| | 10:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=16 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 10(GETNEXT)
+| |
+| 09:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+| HDFS partitions=1824/1824 files=1824 size=389.90GB
+| runtime filters: RF015[min_max] -> ss_customer_sk, RF016[bloom] -> ss_sold_date_sk, RF014[bloom] -> ss_customer_sk
+| stored statistics:
+| table: rows=8.64G size=389.90GB
+| partitions: 1824/1824 rows=8.64G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
+| mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+| tuple-ids=15 row-size=24B cardinality=1.77G(filtered from 8.64G) cost=610616051
+| in pipelines: 09(GETNEXT)
+|
+58:EXCHANGE [HASH(customer_id)]
+| mem-estimate=15.62MB mem-reservation=0B thread-reservation=0
+| tuple-ids=12 row-size=44B cardinality=162.19M cost=50846587
+| in pipelines: 52(GETNEXT)
+|
+F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=2.33GB mem-reservation=34.00MB thread-reservation=1
+max-parallelism=120 segment-costs=[11015581984, 569932672] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+00:UNION
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=12 row-size=44B cardinality=162.19M cost=26790556
+| in pipelines: 52(GETNEXT)
+|
+52:AGGREGATE [FINALIZE]
+| output: sum:merge(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / 2)
+| group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| having: sum(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / 2) > CAST(0 AS DECIMAL(3,0))
+| mem-estimate=2.28GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=169B cardinality=162.19M cost=9573431219
+| in pipelines: 52(GETNEXT), 02(OPEN)
+|
+51:EXCHANGE [HASH(c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year)]
+| mem-estimate=30.29MB mem-reservation=0B thread-reservation=0
+| tuple-ids=3 row-size=169B cardinality=1.62G cost=1442150765
+| in pipelines: 02(GETNEXT)
+|
+F03:PLAN FRAGMENT [HASH(ss_customer_sk)] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=1.80GB mem-reservation=34.00MB thread-reservation=1
+max-parallelism=1860 segment-costs=[10841902425, 18503009586] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+06:AGGREGATE [STREAMING]
+| output: sum(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / CAST(2 AS DECIMAL(3,0)))
+| group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| mem-estimate=1.71GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=169B cardinality=1.62G cost=9573431219
+| in pipelines: 02(GETNEXT)
+|
+05:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=15
+| hash predicates: ss_customer_sk = c_customer_sk
+| fk/pk conjuncts: ss_customer_sk = c_customer_sk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1,2,0 row-size=185B cardinality=1.62G cost=747080012
+| in pipelines: 02(GETNEXT), 01(OPEN)
+|
+|--F53:PLAN FRAGMENT [HASH(ss_customer_sk)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=70.45MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | max-parallelism=1860 segment-costs=[54467197]
+| JOIN BUILD
+| | join-table-id=15 plan-id=16 cohort-id=01
+| | build expressions: c_customer_sk
+| | runtime filters: RF010[bloom] <- c_customer_sk, RF011[min_max] <- c_customer_sk
+| | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| |
+| 50:EXCHANGE [HASH(c_customer_sk)]
+| | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=0 row-size=153B cardinality=30.00M cost=24467197
+| | in pipelines: 01(GETNEXT)
+| |
+| F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| max-parallelism=10 segment-costs=[341205999]
+| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| HDFS partitions=1/1 files=1 size=1.55GB
+| runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF007[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF009[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| stored statistics:
+| table: rows=30.00M size=1.55GB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.10M
+| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| tuple-ids=0 row-size=153B cardinality=30.00M cost=29871228
+| in pipelines: 01(GETNEXT)
+|
+49:EXCHANGE [HASH(ss_customer_sk)]
+| mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| tuple-ids=1,2 row-size=32B cardinality=1.77G cost=521391194
+| in pipelines: 02(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Host Shared Resources: mem-estimate=17.00MB mem-reservation=17.00MB thread-reservation=0 runtime-filters-memory=17.00MB
+Per-Instance Resources: mem-estimate=34.75MB mem-reservation=8.00MB thread-reservation=1
+max-parallelism=374 segment-costs=[6845578389]
+04:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=16
+| hash predicates: ss_sold_date_sk = d_date_sk
+| fk/pk conjuncts: ss_sold_date_sk = d_date_sk
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1,2 row-size=32B cardinality=1.77G cost=773340988
+| in pipelines: 02(GETNEXT), 03(OPEN)
+|
+|--F54:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[863]
+| JOIN BUILD
+| | join-table-id=16 plan-id=17 cohort-id=01
+| | build expressions: d_date_sk
+| | runtime filters: RF012[bloom] <- d_date_sk
+| | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| |
+| 48:EXCHANGE [BROADCAST]
+| | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=2 row-size=8B cardinality=373 cost=490
+| | in pipelines: 03(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| max-parallelism=1 segment-costs=[10483]
+| 85:TUPLE CACHE
+| | cache key: 09df1e7b5f44dfdb1fd5f1f4c9cf3018
+| | input scan node ids: 3
+| | estimated serialized size: 4.37KB
+| | estimated serialized size per node: 4.37KB
+| | cumulative processing cost: 10467
+| | cache read processing cost: 49
+| | cache write processing cost: 12
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=2 row-size=8B cardinality=373 cost=0
+| | in pipelines: 03(GETNEXT)
+| |
+| 03:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| HDFS partitions=1/1 files=1 size=2.17MB
+| deterministic scan range assignment: true
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| stored statistics:
+| table: rows=73.05K size=2.17MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=2 row-size=8B cardinality=373 cost=10467
+| in pipelines: 03(GETNEXT)
+|
+02:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ runtime filters: RF011[min_max] -> ss_customer_sk, RF012[bloom] -> ss_sold_date_sk, RF010[bloom] -> ss_customer_sk
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=1 row-size=24B cardinality=1.77G(filtered from 8.64G) cost=610616051
+ in pipelines: 02(GETNEXT)
+---- PARALLELPLANS
+Max Per-Host Resource Reservation: Memory=10.37GB Threads=379
+Per-Host Resource Estimates: Memory=182.31GB
+F37:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=5.02MB mem-reservation=4.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[442] cpu-comparison-result=1440 [max(1 (self) vs 1440 (sum children))]
+PLAN-ROOT SINK
+| output exprs: customer_id, customer_first_name, customer_last_name, customer_birth_country
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=400
+|
+84:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: customer_id ASC, customer_first_name ASC, customer_last_name ASC, customer_birth_country ASC
+| limit: 100
+| mem-estimate=1.02MB mem-reservation=0B thread-reservation=0
+| tuple-ids=84 row-size=85B cardinality=100 cost=42
+| in pipelines: 47(GETNEXT)
+|
+F12:PLAN FRAGMENT [HASH(customer_id)] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=15.63MB mem-reservation=0B thread-reservation=1
+max-parallelism=1030 segment-costs=[10231383661, 253] cpu-comparison-result=1440 [max(120 (self) vs 1440 (sum children))]
+47:TOP-N [LIMIT=100]
+| order by: customer_id ASC, customer_first_name ASC, customer_last_name ASC, customer_birth_country ASC
+| mem-estimate=8.27KB mem-reservation=0B thread-reservation=0
+| tuple-ids=84 row-size=85B cardinality=100 cost=9722546971
+| in pipelines: 47(GETNEXT), 52(OPEN)
+|
+46:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=00
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| other predicates: CASE WHEN year_total > CAST(0 AS DECIMAL(3,0)) THEN year_total / year_total ELSE NULL END > CASE WHEN year_total > CAST(0 AS DECIMAL(3,0)) THEN year_total / year_total ELSE NULL END, CASE WHEN year_total > CAST(0 AS DECIMAL(3,0)) THEN year_total / year_total ELSE NULL END > CASE WHEN year_total > CAST(0 AS DECIMAL(3,0)) THEN year_total / year_total ELSE NULL END
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26,54,82,68,40 row-size=321B cardinality=591.78M cost=163597061
+| in pipelines: 52(GETNEXT), 82(OPEN)
+|
+|--F38:PLAN FRAGMENT [HASH(customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=63.13MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=1030 segment-costs=[106113816] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: customer_id
+| | runtime filters: RF001[min_max] <- customer_id
+| | mem-estimate=47.51MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=80787070
+| |
+| 83:EXCHANGE [HASH(customer_id)]
+| | mem-estimate=15.62MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=40 row-size=44B cardinality=80.79M cost=25326746
+| | in pipelines: 82(GETNEXT)
+| |
+| F36:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.19GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[5486874642, 283884148] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 14:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=40 row-size=44B cardinality=80.79M cost=13344408
+| | in pipelines: 82(GETNEXT)
+| |
+| 82:AGGREGATE [FINALIZE]
+| | output: sum:merge((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / 2))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | having: sum((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / 2)) > CAST(0 AS DECIMAL(3,0))
+| | mem-estimate=1.14GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=35 row-size=169B cardinality=80.79M cost=4768537611
+| | in pipelines: 82(GETNEXT), 16(OPEN)
+| |
+| 81:EXCHANGE [HASH(c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year)]
+| | mem-estimate=30.29MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=35 row-size=169B cardinality=807.87M cost=718337031
+| | in pipelines: 16(GETNEXT)
+| |
+| F34:PLAN FRAGMENT [HASH(cs_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=968.12MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=930 segment-costs=[5400364645, 9216371342] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 20:AGGREGATE [STREAMING]
+| | output: sum((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / CAST(2 AS DECIMAL(3,0))))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=872.27MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=35 row-size=169B cardinality=807.87M cost=4768537611
+| | in pipelines: 16(GETNEXT)
+| |
+| 19:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=01
+| | hash predicates: cs_bill_customer_sk = c_customer_sk
+| | fk/pk conjuncts: cs_bill_customer_sk = c_customer_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=33,34,32 row-size=185B cardinality=807.87M cost=372121453
+| | in pipelines: 16(GETNEXT), 15(OPEN)
+| |
+| |--F39:PLAN FRAGMENT [HASH(cs_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=70.45MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | | max-parallelism=930 segment-costs=[54467197]
+| | JOIN BUILD
+| | | join-table-id=01 plan-id=02 cohort-id=02
+| | | build expressions: c_customer_sk
+| | | runtime filters: RF030[bloom] <- c_customer_sk, RF031[min_max] <- c_customer_sk
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | |
+| | 80:EXCHANGE [HASH(c_customer_sk)]
+| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=32 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 15(GETNEXT)
+| | |
+| | F33:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[341205999]
+| | 15:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=32 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 15(GETNEXT)
+| |
+| 79:EXCHANGE [HASH(cs_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=33,34 row-size=32B cardinality=880.06M cost=259705581
+| | in pipelines: 16(GETNEXT)
+| |
+| F31:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=17.00MB mem-reservation=17.00MB thread-reservation=0 runtime-filters-memory=17.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=1.00MB thread-reservation=1
+| max-parallelism=350 segment-costs=[3409790834]
+| 18:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=02
+| | hash predicates: cs_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: cs_sold_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=33,34 row-size=32B cardinality=880.06M cost=385202077
+| | in pipelines: 16(GETNEXT), 17(OPEN)
+| |
+| |--F40:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=02 plan-id=03 cohort-id=02
+| | | build expressions: d_date_sk
+| | | runtime filters: RF032[bloom] <- d_date_sk
+| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 78:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=34 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 17(GETNEXT)
+| | |
+| | F32:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 90:TUPLE CACHE
+| | | cache key: 09df1e7b5f44dfdb1fd5f1f4c9cf3018
+| | | input scan node ids: 17
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=34 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 17(GETNEXT)
+| | |
+| | 17:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=34 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 17(GETNEXT)
+| |
+| 16:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
+| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| runtime filters: RF031[min_max] -> cs_bill_customer_sk, RF032[bloom] -> cs_sold_date_sk, RF030[bloom] -> cs_bill_customer_sk
+| stored statistics:
+| table: rows=4.32G size=280.96GB
+| partitions: 1831/1831 rows=4.32G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=21.52M est-scan-range=374(filtered from 1831)
+| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| tuple-ids=33 row-size=24B cardinality=880.06M(filtered from 4.32G) cost=304148590
+| in pipelines: 16(GETNEXT)
+|
+45:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=03
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26,54,82,68 row-size=277B cardinality=219.75M cost=81421260
+| in pipelines: 52(GETNEXT), 76(OPEN)
+|
+|--F41:PLAN FRAGMENT [HASH(customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=49.62MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=1030 segment-costs=[53390593] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| JOIN BUILD
+| | join-table-id=03 plan-id=04 cohort-id=01
+| | build expressions: customer_id
+| | runtime filters: RF003[min_max] <- customer_id
+| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=40647578
+| |
+| 77:EXCHANGE [HASH(customer_id)]
+| | mem-estimate=15.62MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=68 row-size=44B cardinality=40.65M cost=12743015
+| | in pipelines: 76(GETNEXT)
+| |
+| F30:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=637.96MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[2760691322, 142834775] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 28:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=68 row-size=44B cardinality=40.65M cost=6714166
+| | in pipelines: 76(GETNEXT)
+| |
+| 76:AGGREGATE [FINALIZE]
+| | output: sum:merge((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / 2))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | having: sum((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / 2)) > CAST(0 AS DECIMAL(3,0))
+| | mem-estimate=585.17MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=67 row-size=169B cardinality=40.65M cost=2399263927
+| | in pipelines: 76(GETNEXT), 30(OPEN)
+| |
+| 75:EXCHANGE [HASH(c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year)]
+| | mem-estimate=30.29MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=67 row-size=169B cardinality=406.48M cost=361427395
+| | in pipelines: 30(GETNEXT)
+| |
+| F28:PLAN FRAGMENT [HASH(ws_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=534.73MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=470 segment-costs=[2716561250, 4637167430] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 34:AGGREGATE [STREAMING]
+| | output: sum((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / CAST(2 AS DECIMAL(3,0))))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=438.88MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=67 row-size=169B cardinality=406.48M cost=2399263927
+| | in pipelines: 30(GETNEXT)
+| |
+| 33:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=04
+| | hash predicates: ws_bill_customer_sk = c_customer_sk
+| | fk/pk conjuncts: ws_bill_customer_sk = c_customer_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=65,66,64 row-size=185B cardinality=406.48M cost=186950480
+| | in pipelines: 30(GETNEXT), 29(OPEN)
+| |
+| |--F42:PLAN FRAGMENT [HASH(ws_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=54.45MB mem-reservation=34.00MB thread-reservation=1
+| | | max-parallelism=470 segment-costs=[54467197]
+| | JOIN BUILD
+| | | join-table-id=04 plan-id=05 cohort-id=03
+| | | build expressions: c_customer_sk
+| | | runtime filters: RF027[min_max] <- c_customer_sk
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | |
+| | 74:EXCHANGE [HASH(c_customer_sk)]
+| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=64 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 29(GETNEXT)
+| | |
+| | F27:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[341205999]
+| | 29:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=64 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 29(GETNEXT)
+| |
+| 73:EXCHANGE [HASH(ws_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=65,66 row-size=32B cardinality=441.70M cost=130346843
+| | in pipelines: 30(GETNEXT)
+| |
+| F25:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=180 segment-costs=[1711382054]
+| 32:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=05
+| | hash predicates: ws_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: ws_sold_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=65,66 row-size=32B cardinality=441.70M cost=193333830
+| | in pipelines: 30(GETNEXT), 31(OPEN)
+| |
+| |--F43:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=05 plan-id=06 cohort-id=03
+| | | build expressions: d_date_sk
+| | | runtime filters: RF028[bloom] <- d_date_sk
+| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 72:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=66 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 31(GETNEXT)
+| | |
+| | F26:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 89:TUPLE CACHE
+| | | cache key: 09df1e7b5f44dfdb1fd5f1f4c9cf3018
+| | | input scan node ids: 31
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=66 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 31(GETNEXT)
+| | |
+| | 31:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=66 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 31(GETNEXT)
+| |
+| 30:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
+| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| runtime filters: RF027[min_max] -> ws_bill_customer_sk, RF028[bloom] -> ws_sold_date_sk
+| stored statistics:
+| table: rows=2.16G size=145.75GB
+| partitions: 1824/1824 rows=2.16G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=374(filtered from 1824)
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=65 row-size=24B cardinality=441.70M(filtered from 2.16G) cost=152652894
+| in pipelines: 30(GETNEXT)
+|
+44:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=06
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26,54,82 row-size=233B cardinality=162.19M cost=70990594
+| in pipelines: 52(GETNEXT), 70(OPEN)
+|
+|--F44:PLAN FRAGMENT [HASH(customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=49.62MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=1030 segment-costs=[39405000] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| JOIN BUILD
+| | join-table-id=06 plan-id=07 cohort-id=01
+| | build expressions: customer_id
+| | runtime filters: RF005[min_max] <- customer_id
+| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| |
+| 71:EXCHANGE [HASH(customer_id)]
+| | mem-estimate=15.62MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=82 row-size=44B cardinality=30.00M cost=9405000
+| | in pipelines: 70(GETNEXT)
+| |
+| F24:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=606.14MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[1005785180, 105419400] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 35:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=82 row-size=44B cardinality=30.00M cost=4955400
+| | in pipelines: 70(GETNEXT)
+| |
+| 70:AGGREGATE [FINALIZE]
+| | output: sum:merge((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / 2))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=553.34MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=81 row-size=169B cardinality=30.00M cost=664015288
+| | in pipelines: 70(GETNEXT), 37(OPEN)
+| |
+| 69:EXCHANGE [HASH(c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year)]
+| | mem-estimate=30.29MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=81 row-size=169B cardinality=384.37M cost=341769892
+| | in pipelines: 37(GETNEXT)
+| |
+| F22:PLAN FRAGMENT [HASH(ws_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=534.73MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=440 segment-costs=[2616446902, 4384958733] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 41:AGGREGATE [STREAMING]
+| | output: sum((((ws_ext_list_price - ws_ext_wholesale_cost - ws_ext_discount_amt) + ws_ext_sales_price) / CAST(2 AS DECIMAL(3,0))))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=438.88MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=81 row-size=169B cardinality=384.37M cost=2299149579
+| | in pipelines: 37(GETNEXT)
+| |
+| 40:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=07
+| | hash predicates: ws_bill_customer_sk = c_customer_sk
+| | fk/pk conjuncts: ws_bill_customer_sk = c_customer_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=79,80,78 row-size=185B cardinality=406.48M cost=186950480
+| | in pipelines: 37(GETNEXT), 36(OPEN)
+| |
+| |--F45:PLAN FRAGMENT [HASH(ws_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=54.45MB mem-reservation=34.00MB thread-reservation=1
+| | | max-parallelism=440 segment-costs=[54467197]
+| | JOIN BUILD
+| | | join-table-id=07 plan-id=08 cohort-id=04
+| | | build expressions: c_customer_sk
+| | | runtime filters: RF023[min_max] <- c_customer_sk
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | |
+| | 68:EXCHANGE [HASH(c_customer_sk)]
+| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=78 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 36(GETNEXT)
+| | |
+| | F21:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[341205999]
+| | 36:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | runtime filters: RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=78 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 36(GETNEXT)
+| |
+| 67:EXCHANGE [HASH(ws_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=79,80 row-size=32B cardinality=441.70M cost=130346843
+| | in pipelines: 37(GETNEXT)
+| |
+| F19:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=180 segment-costs=[1711382054]
+| 39:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=08
+| | hash predicates: ws_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: ws_sold_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=79,80 row-size=32B cardinality=441.70M cost=193333830
+| | in pipelines: 37(GETNEXT), 38(OPEN)
+| |
+| |--F46:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=08 plan-id=09 cohort-id=04
+| | | build expressions: d_date_sk
+| | | runtime filters: RF024[bloom] <- d_date_sk
+| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 66:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=80 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 38(GETNEXT)
+| | |
+| | F20:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 88:TUPLE CACHE
+| | | cache key: 7b1386755ea9bf5bc7f77efa229d8f45
+| | | input scan node ids: 38
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=80 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 38(GETNEXT)
+| | |
+| | 38:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=80 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 38(GETNEXT)
+| |
+| 37:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
+| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| runtime filters: RF023[min_max] -> ws_bill_customer_sk, RF024[bloom] -> ws_sold_date_sk
+| stored statistics:
+| table: rows=2.16G size=145.75GB
+| partitions: 1824/1824 rows=2.16G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=374(filtered from 1824)
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=79 row-size=24B cardinality=441.70M(filtered from 2.16G) cost=152652894
+| in pipelines: 37(GETNEXT)
+|
+43:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=09
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26,54 row-size=189B cardinality=162.19M cost=70990594
+| in pipelines: 52(GETNEXT), 64(OPEN)
+|
+|--F47:PLAN FRAGMENT [HASH(customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=49.62MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=1030 segment-costs=[39405000] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| JOIN BUILD
+| | join-table-id=09 plan-id=10 cohort-id=01
+| | build expressions: customer_id
+| | runtime filters: RF007[min_max] <- customer_id
+| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| |
+| 65:EXCHANGE [HASH(customer_id)]
+| | mem-estimate=15.62MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=54 row-size=44B cardinality=30.00M cost=9405000
+| | in pipelines: 64(GETNEXT)
+| |
+| F18:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.07GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[1773652148, 105419400] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 21:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=54 row-size=44B cardinality=30.00M cost=4955400
+| | in pipelines: 64(GETNEXT)
+| |
+| 64:AGGREGATE [FINALIZE]
+| | output: sum:merge((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / 2))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=1.02GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=49 row-size=169B cardinality=30.00M cost=1130209988
+| | in pipelines: 64(GETNEXT), 23(OPEN)
+| |
+| 63:EXCHANGE [HASH(c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year)]
+| | mem-estimate=30.29MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=49 row-size=169B cardinality=723.64M cost=643442160
+| | in pipelines: 23(GETNEXT)
+| |
+| F16:PLAN FRAGMENT [HASH(cs_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=968.12MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=830 segment-costs=[5018930073, 8255458964] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 27:AGGREGATE [STREAMING]
+| | output: sum((((cs_ext_list_price - cs_ext_wholesale_cost - cs_ext_discount_amt) + cs_ext_sales_price) / CAST(2 AS DECIMAL(3,0))))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=872.27MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=49 row-size=169B cardinality=723.64M cost=4387103039
+| | in pipelines: 23(GETNEXT)
+| |
+| 26:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=10
+| | hash predicates: cs_bill_customer_sk = c_customer_sk
+| | fk/pk conjuncts: cs_bill_customer_sk = c_customer_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=47,48,46 row-size=185B cardinality=807.87M cost=372121453
+| | in pipelines: 23(GETNEXT), 22(OPEN)
+| |
+| |--F48:PLAN FRAGMENT [HASH(cs_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=70.45MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | | max-parallelism=830 segment-costs=[54467197]
+| | JOIN BUILD
+| | | join-table-id=10 plan-id=11 cohort-id=05
+| | | build expressions: c_customer_sk
+| | | runtime filters: RF018[bloom] <- c_customer_sk, RF019[min_max] <- c_customer_sk
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | |
+| | 62:EXCHANGE [HASH(c_customer_sk)]
+| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=46 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 22(GETNEXT)
+| | |
+| | F15:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[341205999]
+| | 22:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=46 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 22(GETNEXT)
+| |
+| 61:EXCHANGE [HASH(cs_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=47,48 row-size=32B cardinality=880.06M cost=259705581
+| | in pipelines: 23(GETNEXT)
+| |
+| F13:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=17.00MB mem-reservation=17.00MB thread-reservation=0 runtime-filters-memory=17.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=1.00MB thread-reservation=1
+| max-parallelism=350 segment-costs=[3409790834]
+| 25:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=11
+| | hash predicates: cs_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: cs_sold_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=47,48 row-size=32B cardinality=880.06M cost=385202077
+| | in pipelines: 23(GETNEXT), 24(OPEN)
+| |
+| |--F49:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=11 plan-id=12 cohort-id=05
+| | | build expressions: d_date_sk
+| | | runtime filters: RF020[bloom] <- d_date_sk
+| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 60:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=48 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 24(GETNEXT)
+| | |
+| | F14:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 87:TUPLE CACHE
+| | | cache key: 7b1386755ea9bf5bc7f77efa229d8f45
+| | | input scan node ids: 24
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=48 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 24(GETNEXT)
+| | |
+| | 24:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=48 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 24(GETNEXT)
+| |
+| 23:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
+| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| runtime filters: RF019[min_max] -> cs_bill_customer_sk, RF020[bloom] -> cs_sold_date_sk, RF018[bloom] -> cs_bill_customer_sk
+| stored statistics:
+| table: rows=4.32G size=280.96GB
+| partitions: 1831/1831 rows=4.32G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=21.52M est-scan-range=374(filtered from 1831)
+| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| tuple-ids=47 row-size=24B cardinality=880.06M(filtered from 4.32G) cost=304148590
+| in pipelines: 23(GETNEXT)
+|
+42:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=12
+| hash predicates: customer_id = customer_id
+| fk/pk conjuncts: assumed fk/pk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=12,26 row-size=145B cardinality=162.19M cost=70990594
+| in pipelines: 52(GETNEXT), 57(OPEN)
+|
+|--F50:PLAN FRAGMENT [HASH(customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=56.27MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=1030 segment-costs=[47226566] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| JOIN BUILD
+| | join-table-id=12 plan-id=13 cohort-id=01
+| | build expressions: customer_id
+| | runtime filters: RF009[min_max] <- customer_id
+| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| |
+| 59:EXCHANGE [HASH(customer_id)]
+| | mem-estimate=22.27MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=26 row-size=101B cardinality=30.00M cost=17226566
+| | in pipelines: 57(GETNEXT)
+| |
+| F11:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.91GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=120 segment-costs=[3091124881, 220311407] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 07:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=26 row-size=101B cardinality=30.00M cost=10345479
+| | in pipelines: 57(GETNEXT)
+| |
+| 57:AGGREGATE [FINALIZE]
+| | output: sum:merge(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / 2)
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=1.84GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=17 row-size=169B cardinality=30.00M cost=1930086569
+| | in pipelines: 57(GETNEXT), 09(OPEN)
+| |
+| 56:EXCHANGE [HASH(c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year)]
+| | mem-estimate=30.29MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=17 row-size=169B cardinality=1.31G cost=1161038312
+| | in pipelines: 09(GETNEXT)
+| |
+| F09:PLAN FRAGMENT [HASH(ss_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.80GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=1490 segment-costs=[9410215497, 14896294857] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 13:AGGREGATE [STREAMING]
+| | output: sum(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / CAST(2 AS DECIMAL(3,0)))
+| | group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| | mem-estimate=1.71GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=17 row-size=169B cardinality=1.31G cost=8141744291
+| | in pipelines: 09(GETNEXT)
+| |
+| 12:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=13
+| | hash predicates: ss_customer_sk = c_customer_sk
+| | fk/pk conjuncts: ss_customer_sk = c_customer_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=15,16,14 row-size=185B cardinality=1.62G cost=747080012
+| | in pipelines: 09(GETNEXT), 08(OPEN)
+| |
+| |--F51:PLAN FRAGMENT [HASH(ss_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=70.45MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | | max-parallelism=1490 segment-costs=[54467197]
+| | JOIN BUILD
+| | | join-table-id=13 plan-id=14 cohort-id=06
+| | | build expressions: c_customer_sk
+| | | runtime filters: RF014[bloom] <- c_customer_sk, RF015[min_max] <- c_customer_sk
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | |
+| | 55:EXCHANGE [HASH(c_customer_sk)]
+| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=14 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 08(GETNEXT)
+| | |
+| | F08:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[341205999]
+| | 08:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF007[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=14 row-size=153B cardinality=30.00M cost=29871228
+| | in pipelines: 08(GETNEXT)
+| |
+| 54:EXCHANGE [HASH(ss_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=15,16 row-size=32B cardinality=1.77G cost=521391194
+| | in pipelines: 09(GETNEXT)
+| |
+| F06:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=17.00MB mem-reservation=17.00MB thread-reservation=0 runtime-filters-memory=17.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=8.00MB thread-reservation=1
+| max-parallelism=374 segment-costs=[6845578389]
+| 11:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=14
+| | hash predicates: ss_sold_date_sk = d_date_sk
+| | fk/pk conjuncts: ss_sold_date_sk = d_date_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=15,16 row-size=32B cardinality=1.77G cost=773340988
+| | in pipelines: 09(GETNEXT), 10(OPEN)
+| |
+| |--F52:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | max-parallelism=10 segment-costs=[863]
+| | JOIN BUILD
+| | | join-table-id=14 plan-id=15 cohort-id=06
+| | | build expressions: d_date_sk
+| | | runtime filters: RF016[bloom] <- d_date_sk
+| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | |
+| | 53:EXCHANGE [BROADCAST]
+| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=16 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 10(GETNEXT)
+| | |
+| | F07:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| | max-parallelism=1 segment-costs=[10483]
+| | 86:TUPLE CACHE
+| | | cache key: 7b1386755ea9bf5bc7f77efa229d8f45
+| | | input scan node ids: 10
+| | | estimated serialized size: 4.37KB
+| | | estimated serialized size per node: 4.37KB
+| | | cumulative processing cost: 10467
+| | | cache read processing cost: 49
+| | | cache write processing cost: 12
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=16 row-size=8B cardinality=373 cost=0
+| | | in pipelines: 10(GETNEXT)
+| | |
+| | 10:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=16 row-size=8B cardinality=373 cost=10467
+| | in pipelines: 10(GETNEXT)
+| |
+| 09:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+| HDFS partitions=1824/1824 files=1824 size=389.90GB
+| runtime filters: RF015[min_max] -> ss_customer_sk, RF016[bloom] -> ss_sold_date_sk, RF014[bloom] -> ss_customer_sk
+| stored statistics:
+| table: rows=8.64G size=389.90GB
+| partitions: 1824/1824 rows=8.64G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
+| mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+| tuple-ids=15 row-size=24B cardinality=1.77G(filtered from 8.64G) cost=610616051
+| in pipelines: 09(GETNEXT)
+|
+58:EXCHANGE [HASH(customer_id)]
+| mem-estimate=15.62MB mem-reservation=0B thread-reservation=0
+| tuple-ids=12 row-size=44B cardinality=162.19M cost=50846587
+| in pipelines: 52(GETNEXT)
+|
+F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=2.33GB mem-reservation=34.00MB thread-reservation=1
+max-parallelism=120 segment-costs=[11015581984, 569932672] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+00:UNION
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=12 row-size=44B cardinality=162.19M cost=26790556
+| in pipelines: 52(GETNEXT)
+|
+52:AGGREGATE [FINALIZE]
+| output: sum:merge(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / 2)
+| group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| having: sum(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / 2) > CAST(0 AS DECIMAL(3,0))
+| mem-estimate=2.28GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=169B cardinality=162.19M cost=9573431219
+| in pipelines: 52(GETNEXT), 02(OPEN)
+|
+51:EXCHANGE [HASH(c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year)]
+| mem-estimate=30.29MB mem-reservation=0B thread-reservation=0
+| tuple-ids=3 row-size=169B cardinality=1.62G cost=1442150765
+| in pipelines: 02(GETNEXT)
+|
+F03:PLAN FRAGMENT [HASH(ss_customer_sk)] hosts=10 instances=120
+Per-Instance Resources: mem-estimate=1.80GB mem-reservation=34.00MB thread-reservation=1
+max-parallelism=1860 segment-costs=[10841902425, 18503009586] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+06:AGGREGATE [STREAMING]
+| output: sum(((ss_ext_list_price - ss_ext_wholesale_cost - ss_ext_discount_amt) + ss_ext_sales_price) / CAST(2 AS DECIMAL(3,0)))
+| group by: c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address, d_year
+| mem-estimate=1.71GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=3 row-size=169B cardinality=1.62G cost=9573431219
+| in pipelines: 02(GETNEXT)
+|
+05:HASH JOIN [INNER JOIN, PARTITIONED]
+| hash-table-id=15
+| hash predicates: ss_customer_sk = c_customer_sk
+| fk/pk conjuncts: ss_customer_sk = c_customer_sk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1,2,0 row-size=185B cardinality=1.62G cost=747080012
+| in pipelines: 02(GETNEXT), 01(OPEN)
+|
+|--F53:PLAN FRAGMENT [HASH(ss_customer_sk)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=70.45MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | max-parallelism=1860 segment-costs=[54467197]
+| JOIN BUILD
+| | join-table-id=15 plan-id=16 cohort-id=01
+| | build expressions: c_customer_sk
+| | runtime filters: RF010[bloom] <- c_customer_sk, RF011[min_max] <- c_customer_sk
+| | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| |
+| 50:EXCHANGE [HASH(c_customer_sk)]
+| | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=0 row-size=153B cardinality=30.00M cost=24467197
+| | in pipelines: 01(GETNEXT)
+| |
+| F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| max-parallelism=10 segment-costs=[341205999]
+| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| HDFS partitions=1/1 files=1 size=1.55GB
+| runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF007[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF009[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| stored statistics:
+| table: rows=30.00M size=1.55GB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.10M
+| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| tuple-ids=0 row-size=153B cardinality=30.00M cost=29871228
+| in pipelines: 01(GETNEXT)
+|
+49:EXCHANGE [HASH(ss_customer_sk)]
+| mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| tuple-ids=1,2 row-size=32B cardinality=1.77G cost=521391194
+| in pipelines: 02(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Host Shared Resources: mem-estimate=17.00MB mem-reservation=17.00MB thread-reservation=0 runtime-filters-memory=17.00MB
+Per-Instance Resources: mem-estimate=34.75MB mem-reservation=8.00MB thread-reservation=1
+max-parallelism=374 segment-costs=[6845578389]
+04:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=16
+| hash predicates: ss_sold_date_sk = d_date_sk
+| fk/pk conjuncts: ss_sold_date_sk = d_date_sk
+| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
+| tuple-ids=1,2 row-size=32B cardinality=1.77G cost=773340988
+| in pipelines: 02(GETNEXT), 03(OPEN)
+|
+|--F54:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[863]
+| JOIN BUILD
+| | join-table-id=16 plan-id=17 cohort-id=01
+| | build expressions: d_date_sk
+| | runtime filters: RF012[bloom] <- d_date_sk
+| | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| |
+| 48:EXCHANGE [BROADCAST]
+| | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| | tuple-ids=2 row-size=8B cardinality=373 cost=490
+| | in pipelines: 03(GETNEXT)
+| |
+| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=20.05MB mem-reservation=4.50MB thread-reservation=1
+| max-parallelism=1 segment-costs=[10483]
+| 85:TUPLE CACHE
+| | cache key: 09df1e7b5f44dfdb1fd5f1f4c9cf3018
+| | input scan node ids: 3
+| | estimated serialized size: 4.37KB
+| | estimated serialized size per node: 4.37KB
+| | cumulative processing cost: 10467
+| | cache read processing cost: 49
+| | cache write processing cost: 12
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=2 row-size=8B cardinality=373 cost=0
+| | in pipelines: 03(GETNEXT)
+| |
+| 03:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| HDFS partitions=1/1 files=1 size=2.17MB
+| deterministic scan range assignment: true
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| stored statistics:
+| table: rows=73.05K size=2.17MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(1999 AS INT)
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=2 row-size=8B cardinality=373 cost=10467
+| in pipelines: 03(GETNEXT)
+|
+02:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+ HDFS partitions=1824/1824 files=1824 size=389.90GB
+ runtime filters: RF011[min_max] -> ss_customer_sk, RF012[bloom] -> ss_sold_date_sk, RF010[bloom] -> ss_customer_sk
+ stored statistics:
+ table: rows=8.64G size=389.90GB
+ partitions: 1824/1824 rows=8.64G
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
+ mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
+ tuple-ids=1 row-size=24B cardinality=1.77G(filtered from 8.64G) cost=610616051
+ in pipelines: 02(GETNEXT)
+====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q05.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q05.test
new file mode 100644
index 000000000..bf31c5498
--- /dev/null
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds_tuple_cache/tpcds-q05.test
@@ -0,0 +1,1661 @@
+# TPCDS-Q5
+# start query 5 in stream 0 using template query5.tpl using seed 246622778
+with ssr as
+ (select s_store_id,
+ sum(sales_price) as sales,
+ sum(profit) as profit,
+ sum(return_amt) as `returns`,
+ sum(net_loss) as profit_loss
+ from
+ ( select ss_store_sk as store_sk,
+ ss_sold_date_sk as date_sk,
+ ss_ext_sales_price as sales_price,
+ ss_net_profit as profit,
+ cast(0 as decimal(7,2)) as return_amt,
+ cast(0 as decimal(7,2)) as net_loss
+ from store_sales
+ union all
+ select sr_store_sk as store_sk,
+ sr_returned_date_sk as date_sk,
+ cast(0 as decimal(7,2)) as sales_price,
+ cast(0 as decimal(7,2)) as profit,
+ sr_return_amt as return_amt,
+ sr_net_loss as net_loss
+ from store_returns
+ ) salesreturns,
+ date_dim,
+ store
+ where date_sk = d_date_sk
+ and d_date between cast('2000-08-19' as date)
+ and (cast('2000-08-19' as date) + interval 14 days)
+ and store_sk = s_store_sk
+ group by s_store_id)
+ ,
+ csr as
+ (select cp_catalog_page_id,
+ sum(sales_price) as sales,
+ sum(profit) as profit,
+ sum(return_amt) as `returns`,
+ sum(net_loss) as profit_loss
+ from
+ ( select cs_catalog_page_sk as page_sk,
+ cs_sold_date_sk as date_sk,
+ cs_ext_sales_price as sales_price,
+ cs_net_profit as profit,
+ cast(0 as decimal(7,2)) as return_amt,
+ cast(0 as decimal(7,2)) as net_loss
+ from catalog_sales
+ union all
+ select cr_catalog_page_sk as page_sk,
+ cr_returned_date_sk as date_sk,
+ cast(0 as decimal(7,2)) as sales_price,
+ cast(0 as decimal(7,2)) as profit,
+ cr_return_amount as return_amt,
+ cr_net_loss as net_loss
+ from catalog_returns
+ ) salesreturns,
+ date_dim,
+ catalog_page
+ where date_sk = d_date_sk
+ and d_date between cast('2000-08-19' as date)
+ and (cast('2000-08-19' as date) + interval 14 days)
+ and page_sk = cp_catalog_page_sk
+ group by cp_catalog_page_id)
+ ,
+ wsr as
+ (select web_site_id,
+ sum(sales_price) as sales,
+ sum(profit) as profit,
+ sum(return_amt) as `returns`,
+ sum(net_loss) as profit_loss
+ from
+ ( select ws_web_site_sk as wsr_web_site_sk,
+ ws_sold_date_sk as date_sk,
+ ws_ext_sales_price as sales_price,
+ ws_net_profit as profit,
+ cast(0 as decimal(7,2)) as return_amt,
+ cast(0 as decimal(7,2)) as net_loss
+ from web_sales
+ union all
+ select ws_web_site_sk as wsr_web_site_sk,
+ wr_returned_date_sk as date_sk,
+ cast(0 as decimal(7,2)) as sales_price,
+ cast(0 as decimal(7,2)) as profit,
+ wr_return_amt as return_amt,
+ wr_net_loss as net_loss
+ from web_returns left outer join web_sales on
+ ( wr_item_sk = ws_item_sk
+ and wr_order_number = ws_order_number)
+ ) salesreturns,
+ date_dim,
+ web_site
+ where date_sk = d_date_sk
+ and d_date between cast('2000-08-19' as date)
+ and (cast('2000-08-19' as date) + interval 14 days)
+ and wsr_web_site_sk = web_site_sk
+ group by web_site_id)
+ select channel
+ , id
+ , sum(sales) as sales
+ , sum(`returns`) as `returns`
+ , sum(profit) as profit
+ from
+ (select 'store channel' as channel
+ , 'store' || s_store_id as id
+ , sales
+ , `returns`
+ , (profit - profit_loss) as profit
+ from ssr
+ union all
+ select 'catalog channel' as channel
+ , 'catalog_page' || cp_catalog_page_id as id
+ , sales
+ , `returns`
+ , (profit - profit_loss) as profit
+ from csr
+ union all
+ select 'web channel' as channel
+ , 'web_site' || web_site_id as id
+ , sales
+ , `returns`
+ , (profit - profit_loss) as profit
+ from wsr
+ ) x
+ group by rollup (channel, id)
+ order by channel
+ ,id
+ limit 100;
+
+# end query 5 in stream 0 using template query5.tpl
+---- PLAN
+Max Per-Host Resource Reservation: Memory=94.64MB Threads=1
+Per-Host Resource Estimates: Memory=10.97GB
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=10.97GB mem-reservation=94.64MB thread-reservation=1 runtime-filters-memory=23.00MB
+| max-parallelism=1 segment-costs=[5637146480, 2816221067, 2740469898, 342336, 221854, 313895, 500]
+PLAN-ROOT SINK
+| output exprs: CASE valid_tid(27,28,29) WHEN 27 THEN channel WHEN 28 THEN channel WHEN 29 THEN NULL END, CASE valid_tid(27,28,29) WHEN 27 THEN id WHEN 28 THEN NULL WHEN 29 THEN NULL END, aggif(valid_tid(27,28,29) IN (27, 28, 29), CASE valid_tid(27,28,29) WHEN 27 THEN sum(sales) WHEN 28 THEN sum(sales) WHEN 29 THEN sum(sales) END), aggif(valid_tid(27,28,29) IN (27, 28, 29), CASE valid_tid(27,28,29) WHEN 27 THEN sum(`returns`) WHEN 28 THEN sum(`returns`) WHEN 29 THEN sum(`returns`) END), [...]
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=500
+|
+29:TOP-N [LIMIT=100]
+| order by: CASE valid_tid(27,28,29) WHEN 27 THEN channel WHEN 28 THEN channel WHEN 29 THEN NULL END ASC, CASE valid_tid(27,28,29) WHEN 27 THEN id WHEN 28 THEN NULL WHEN 29 THEN NULL END ASC
+| mem-estimate=7.03KB mem-reservation=0B thread-reservation=0
+| tuple-ids=31 row-size=72B cardinality=100 cost=313895
+| in pipelines: 29(GETNEXT), 28(OPEN)
+|
+38:TUPLE CACHE
+| cache key: 4245a052541e2b55ae904c09f9d3fd1c
+| input scan node ids: 2,3,4,5,10,11,12,13,18,20,19,22,23
+| estimated serialized size: 2.80MB
+| estimated serialized size per node: 286.84KB
+| cumulative processing cost: 11194401635
+| cache read processing cost: 4879
+| cache write processing cost: 7930
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=30 row-size=76B cardinality=36.72K cost=0
+| in pipelines: 28(GETNEXT)
+|
+28:AGGREGATE [FINALIZE]
+| output: aggif(valid_tid(27,28,29) IN (CAST(27 AS INT), CAST(28 AS INT), CAST(29 AS INT)), CASE valid_tid(27,28,29) WHEN CAST(27 AS INT) THEN sum(sales) WHEN CAST(28 AS INT) THEN sum(sales) WHEN CAST(29 AS INT) THEN sum(sales) END), aggif(valid_tid(27,28,29) IN (CAST(27 AS INT), CAST(28 AS INT), CAST(29 AS INT)), CASE valid_tid(27,28,29) WHEN CAST(27 AS INT) THEN sum(`returns`) WHEN CAST(28 AS INT) THEN sum(`returns`) WHEN CAST(29 AS INT) THEN sum(`returns`) END), aggif(valid_tid(27,28 [...]
+| group by: CASE valid_tid(27,28,29) WHEN CAST(27 AS INT) THEN channel WHEN CAST(28 AS INT) THEN channel WHEN CAST(29 AS INT) THEN NULL END, CASE valid_tid(27,28,29) WHEN CAST(27 AS INT) THEN id WHEN CAST(28 AS INT) THEN NULL WHEN CAST(29 AS INT) THEN NULL END, CASE valid_tid(27,28,29) WHEN CAST(27 AS INT) THEN CAST(27 AS INT) WHEN CAST(28 AS INT) THEN CAST(28 AS INT) WHEN CAST(29 AS INT) THEN CAST(29 AS INT) END
+| mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB thread-reservation=0
+| tuple-ids=30 row-size=76B cardinality=36.72K cost=221854
+| in pipelines: 28(GETNEXT), 27(OPEN)
+|
+37:TUPLE CACHE
+| cache key: d06a88f260e92d8b2da29236eccc8aae
+| input scan node ids: 2,3,4,5,10,11,12,13,18,20,19,22,23
+| estimated serialized size: 7.98MB
+| estimated serialized size per node: 817.48KB
+| cumulative processing cost: 11194179781
+| cache read processing cost: 4879
+| cache write processing cost: 22601
+| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=27N,28N,29N row-size=216B cardinality=36.72K cost=0
+| in pipelines: 27(GETNEXT)
+|
+27:AGGREGATE [FINALIZE]
+| Class 0
+| output: sum(sales), sum(returns), sum(profit)
+| group by: channel, id
+| Class 1
+| output: sum(sales), sum(returns), sum(profit)
+| group by: channel, NULL
+| Class 2
+| output: sum(sales), sum(returns), sum(profit)
+| group by: NULL, NULL
+| mem-estimate=30.00MB mem-reservation=12.38MB thread-reservation=0
+| tuple-ids=27N,28N,29N row-size=216B cardinality=36.72K cost=333014
+| in pipelines: 27(GETNEXT), 08(OPEN), 16(OPEN), 26(OPEN)
+|
+00:UNION
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+| tuple-ids=25 row-size=72B cardinality=36.71K cost=9322
+| in pipelines: 08(GETNEXT), 16(GETNEXT), 26(GETNEXT)
+|
+|--36:TUPLE CACHE
+| | cache key: 06a62fb778189efed0c65fac5f1e0140
+| | input scan node ids: 18,20,19,22,23
+| | estimated serialized size: 3.09KB
+| | estimated serialized size per node: 316B
+| | cumulative processing cost: 2740469898
+| | cache read processing cost: 4
+| | cache write processing cost: 8
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=23 row-size=92B cardinality=33 cost=0
+| | in pipelines: 26(GETNEXT)
+| |
+| 26:AGGREGATE [FINALIZE]
+| | output: sum(sales_price), sum(profit), sum(return_amt), sum(net_loss)
+| | group by: web_site_id
+| | mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=23 row-size=92B cardinality=33 cost=4479919
+| | in pipelines: 26(GETNEXT), 18(OPEN), 20(OPEN)
+| |
+| 25:HASH JOIN [INNER JOIN]
+| | hash predicates: wsr_web_site_sk = web_site_sk
+| | fk/pk conjuncts: assumed fk/pk
+| | runtime filters: RF008[bloom] <- web_site_sk, RF009[min_max] <- web_site_sk
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=19,21,22 row-size=64B cardinality=8.92M cost=3936848
+| | in pipelines: 18(GETNEXT), 20(GETNEXT), 23(OPEN)
+| |
+| |--23:SCAN HDFS [tpcds_partitioned_parquet_snap.web_site]
+| | HDFS partitions=1/1 files=1 size=17.88KB
+| | deterministic scan range assignment: true
+| | stored statistics:
+| | table: rows=66 size=17.88KB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=66
+| | mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
+| | tuple-ids=22 row-size=32B cardinality=66 cost=19
+| | in pipelines: 23(GETNEXT)
+| |
+| 35:TUPLE CACHE
+| | cache key: e1724cb65660894995b7827a63027dc7
+| | input scan node ids: 18,20,19,22
+| | estimated serialized size: 345.24MB
+| | estimated serialized size per node: 34.52MB
+| | cumulative processing cost: 2732053112
+| | cache read processing cost: 1202767
+| | cache write processing cost: 977418
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=19,21 row-size=32B cardinality=9.05M cost=0
+| | in pipelines: 18(GETNEXT), 20(GETNEXT)
+| |
+| 24:HASH JOIN [INNER JOIN]
+| | hash predicates: date_sk = d_date_sk
+| | fk/pk conjuncts: assumed fk/pk
+| | runtime filters: RF010[bloom] <- d_date_sk
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=19,21 row-size=32B cardinality=9.05M cost=611076377
+| | in pipelines: 18(GETNEXT), 20(GETNEXT), 22(OPEN)
+| |
+| |--34:TUPLE CACHE
+| | | cache key: 83717adeca14e11d2695adfe5bca39c1
+| | | input scan node ids: 22
+| | | estimated serialized size: 180B
+| | | estimated serialized size per node: 180B
+| | | cumulative processing cost: 12520
+| | | cache read processing cost: 1
+| | | cache write processing cost: 0
+| | | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=21 row-size=8B cardinality=15 cost=0
+| | | in pipelines: 22(GETNEXT)
+| | |
+| | 22:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | deterministic scan range assignment: true
+| | predicates: d_date <= DATE '2000-09-02', d_date >= DATE '2000-08-19'
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: d_date <= DATE '2000-09-02', d_date >= DATE '2000-08-19'
+| | parquet dictionary predicates: d_date <= DATE '2000-09-02', d_date >= DATE '2000-08-19'
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=21 row-size=8B cardinality=15 cost=12520
+| | in pipelines: 22(GETNEXT)
+| |
+| 17:UNION
+| | mem-estimate=0B mem-reservation=0B thread-reservation=0
+| | tuple-ids=19 row-size=24B cardinality=2.38G cost=241826293
+| | in pipelines: 18(GETNEXT), 20(GETNEXT)
+| |
+| |--21:HASH JOIN [RIGHT OUTER JOIN]
+| | | hash predicates: ws_item_sk = wr_item_sk, ws_order_number = wr_order_number
+| | | fk/pk conjuncts: ws_item_sk = wr_item_sk, ws_order_number = wr_order_number
+| | | runtime filters: RF012[bloom] <- wr_item_sk, RF013[bloom] <- wr_order_number, RF014[min_max] <- wr_item_sk, RF015[min_max] <- wr_order_number
+| | | mem-estimate=10.83GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=18N,17 row-size=40B cardinality=216.00M cost=809174156
+| | | in pipelines: 20(GETNEXT), 19(OPEN)
+| | |
+| | |--19:SCAN HDFS [tpcds_partitioned_parquet_snap.web_returns]
+| | | HDFS partitions=2114/2114 files=2114 size=16.74GB
+| | | deterministic scan range assignment: true
+| | | runtime filters: RF010[bloom] -> tpcds_partitioned_parquet_snap.web_returns.wr_returned_date_sk
+| | | stored statistics:
+| | | table: rows=216.00M size=16.74GB
+| | | partitions: 2114/2114 rows=216.00M
+| | | columns: all
+| | | extrapolated-rows=disabled max-scan-range-rows=9.41M
+| | | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| | | tuple-ids=17 row-size=24B cardinality=216.00M cost=74650528
+| | | in pipelines: 19(GETNEXT)
+| | |
+| | 20:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
+| | HDFS partitions=1824/1824 files=1824 size=145.75GB
+| | deterministic scan range assignment: true
+| | runtime filters: RF014[min_max] -> ws_item_sk, RF015[min_max] -> ws_order_number, RF009[min_max] -> tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk, RF012[bloom] -> ws_item_sk, RF013[bloom] -> ws_order_number, RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk
+| | stored statistics:
+| | table: rows=2.16G size=145.75GB
+| | partitions: 1824/1824 rows=2.16G
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.37M
+| | mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| | tuple-ids=18 row-size=16B cardinality=2.16G cost=497656619
+| | in pipelines: 20(GETNEXT)
+| |
+| 18:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
+| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| deterministic scan range assignment: true
+| runtime filters: RF009[min_max] -> tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk, RF010[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk, RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk
+| stored statistics:
+| table: rows=2.16G size=145.75GB
+| partitions: 1824/1824 rows=2.16G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.37M
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=16 row-size=16B cardinality=2.16G cost=497656619
+| in pipelines: 18(GETNEXT)
+|
+|--33:TUPLE CACHE
+| | cache key: dfea08ccd64a987b0ebfbabd4abc3c5a
+| | input scan node ids: 10,11,12,13
+| | estimated serialized size: 3.30MB
+| | estimated serialized size per node: 337.50KB
+| | cumulative processing cost: 2816221067
+| | cache read processing cost: 4784
+| | cache write processing cost: 9331
+| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=14 row-size=92B cardinality=36.00K cost=0
+| | in pipelines: 16(GETNEXT)
+| |
+| 16:AGGREGATE [FINALIZE]
+| | output: sum(sales_price), sum(profit), sum(return_amt), sum(net_loss)
+| | group by: cp_catalog_page_id
+| | mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB thread-reservation=0
+| | tuple-ids=14 row-size=92B cardinality=36.00K cost=7874684
+| | in pipelines: 16(GETNEXT), 10(OPEN), 11(OPEN)
+| |
+| 15:HASH JOIN [INNER JOIN]
+| | hash predicates: page_sk = cp_catalog_page_sk
+| | fk/pk conjuncts: assumed fk/pk
+| | runtime filters: RF004[bloom] <- cp_catalog_page_sk, RF005[min_max] <- cp_catalog_page_sk
+| | mem-estimate=2.88MB mem-reservation=2.88MB spill-buffer=128.00KB thread-reservation=0
+| | tuple-ids=10,12,13 row-size=64B cardinality=15.48M cost=7540710
+| | in pipelines: 10(GETNEXT), 11(GETNEXT), 13(OPEN)
+| |
+| |--13:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_page]
+| | HDFS partitions=1/1 files=1 size=2.24MB
+| | deterministic scan range assignment: true
+| | stored statistics:
+| | table: rows=36.00K size=2.24MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=36.00K
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=13 row-size=32B cardinality=36.00K cost=10368
+| | in pipelines: 13(GETNEXT)
+| |
+| 14:HASH JOIN [INNER JOIN]
+| | hash predicates: date_sk = d_date_sk
+| | fk/pk conjuncts: assumed fk/pk
+| | runtime filters: RF006[bloom] <- d_date_sk
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=10,12 row-size=32B cardinality=18.32M cost=1222231879
+| | in pipelines: 10(GETNEXT), 11(GETNEXT), 12(OPEN)
+| |
+| |--32:TUPLE CACHE
+| | | cache key: 83717adeca14e11d2695adfe5bca39c1
+| | | input scan node ids: 12
... 131619 lines suppressed ...
(impala) 02/03: IMPALA-13437 (part 1): Compute processing cost before TupleCachePlanner
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 3181fe18006e392e0ce3f2f48fe285569ccfd148
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Fri Mar 14 09:24:14 2025 -0700
IMPALA-13437 (part 1): Compute processing cost before TupleCachePlanner
This is a preparatory change for cost-based placement for
TupleCacheNodes. It reorders planning so that the processing cost and
filtered cardinality are calculated before running the TupleCachePlanner.
This computes the processing cost when enable_tuple_cache=true.
It also displays the cost information in the explain plan output
when enable_tuple_cache=true. This does not impact the adjustment
of fragment parallelism, which continues to be controlled by the
compute_processing_cost option.
This uses the processing cost to calculate a cumulative processing
cost in the TupleCacheInfo. This is all of the processing cost below
this point including other fragments. This is an indicator of how
much processing a cache hit could avoid. This does not accumulate the
cost when merging the TupleCacheInfo due to a runtime filter, as that
cost is not actually being avoided. This also computes the estimated
serialized size for the TupleCacheNode based on the filtered
cardinality and the row size.
Testing:
- Ran a core job
Change-Id: If78f5d002b0e079eef1eece612f0d4fefde545c7
Reviewed-on: http://gerrit.cloudera.org:8080/23164
Reviewed-by: Yida Wu <wy...@gmail.com>
Reviewed-by: Michael Smith <mi...@cloudera.com>
Tested-by: Michael Smith <mi...@cloudera.com>
---
.../java/org/apache/impala/planner/DataSink.java | 2 +-
.../org/apache/impala/planner/PlanFragment.java | 6 +-
.../java/org/apache/impala/planner/PlanNode.java | 5 +-
.../java/org/apache/impala/planner/Planner.java | 28 ++++++--
.../org/apache/impala/planner/TupleCacheInfo.java | 80 ++++++++++++++++++++++
.../org/apache/impala/planner/TupleCacheNode.java | 18 +++--
.../java/org/apache/impala/service/Frontend.java | 6 +-
7 files changed, 122 insertions(+), 23 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/planner/DataSink.java b/fe/src/main/java/org/apache/impala/planner/DataSink.java
index 97571207d..dfbf9f2d2 100644
--- a/fe/src/main/java/org/apache/impala/planner/DataSink.java
+++ b/fe/src/main/java/org/apache/impala/planner/DataSink.java
@@ -60,7 +60,7 @@ public abstract class DataSink {
if (explainLevel.ordinal() >= TExplainLevel.EXTENDED.ordinal()) {
output.append(detailPrefix);
output.append(resourceProfile_.getExplainString());
- if (queryOptions.isCompute_processing_cost()) {
+ if (Planner.isProcessingCostAvailable(queryOptions)) {
// Show processing cost total.
output.append(" cost=");
if (processingCost_.isValid()) {
diff --git a/fe/src/main/java/org/apache/impala/planner/PlanFragment.java b/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
index 80c9bd979..1589c6710 100644
--- a/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
+++ b/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
@@ -744,14 +744,14 @@ public class PlanFragment extends TreeNode<PlanFragment> {
*/
public String getFragmentHeaderString(String firstLinePrefix, String detailPrefix,
TQueryOptions queryOptions, TExplainLevel explainLevel) {
- boolean isComputeCost = queryOptions.isCompute_processing_cost();
+ boolean adjustsInstanceCount = queryOptions.isCompute_processing_cost();
boolean useMTFragment = Planner.useMTFragment(queryOptions);
StringBuilder builder = new StringBuilder();
builder.append(String.format("%s%s:PLAN FRAGMENT [%s]", firstLinePrefix,
fragmentId_.toString(), dataPartition_.getExplainString()));
builder.append(PrintUtils.printNumHosts(" ", getNumNodes()));
builder.append(PrintUtils.printNumInstances(" ", getNumInstances()));
- if (isComputeCost && originalInstanceCount_ != getNumInstances()) {
+ if (adjustsInstanceCount && originalInstanceCount_ != getNumInstances()) {
builder.append(" (adjusted from " + originalInstanceCount_ + ")");
}
builder.append("\n");
@@ -807,7 +807,7 @@ public class PlanFragment extends TreeNode<PlanFragment> {
builder.append(perInstanceExplainString);
builder.append("\n");
}
- if (isComputeCost && rootSegment_ != null
+ if (Planner.isProcessingCostAvailable(queryOptions) && rootSegment_ != null
&& explainLevel.ordinal() >= TExplainLevel.EXTENDED.ordinal()) {
// Print processing cost.
builder.append(detailPrefix);
diff --git a/fe/src/main/java/org/apache/impala/planner/PlanNode.java b/fe/src/main/java/org/apache/impala/planner/PlanNode.java
index 27116e7a6..acbbbff7f 100644
--- a/fe/src/main/java/org/apache/impala/planner/PlanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/PlanNode.java
@@ -388,7 +388,7 @@ abstract public class PlanNode extends TreeNode<PlanNode> {
expBuilder.append(nodeResourceProfile_.getExplainString());
expBuilder.append("\n");
- if (queryOptions.isCompute_processing_cost() && processingCost_.isValid()
+ if (Planner.isProcessingCostAvailable(queryOptions) && processingCost_.isValid()
&& detailLevel.ordinal() >= TExplainLevel.VERBOSE.ordinal()) {
// Print processing cost.
expBuilder.append(processingCost_.getExplainString(detailPrefix, false));
@@ -421,7 +421,7 @@ abstract public class PlanNode extends TreeNode<PlanNode> {
} else {
expBuilder.append(PrintUtils.printEstCardinality(cardinality_));
}
- if (queryOptions.isCompute_processing_cost()) {
+ if (Planner.isProcessingCostAvailable(queryOptions)) {
// Show processing cost total.
expBuilder.append(" cost=");
if (processingCost_.isValid()) {
@@ -1412,6 +1412,7 @@ abstract public class PlanNode extends TreeNode<PlanNode> {
// Leaf node, add query options hash.
tupleCacheInfo_.hashThrift("Query options hash", queryOptsHash);
}
+ tupleCacheInfo_.calculateCostInformation(this);
tupleCacheInfo_.finalizeHash();
LOG.trace("Hash for {}:", this);
for (HashTraceElement elem : tupleCacheInfo_.getHashTraces()) {
diff --git a/fe/src/main/java/org/apache/impala/planner/Planner.java b/fe/src/main/java/org/apache/impala/planner/Planner.java
index 5c5db03c0..5da40d760 100644
--- a/fe/src/main/java/org/apache/impala/planner/Planner.java
+++ b/fe/src/main/java/org/apache/impala/planner/Planner.java
@@ -166,6 +166,7 @@ public class Planner {
ctx_.getTimeline().markEvent("Runtime filters computed");
checkAndOverrideMinmaxFilterThresholdAndLevel(ctx_.getQueryOptions());
+ reduceCardinalityByRuntimeFilter(rootFragment, ctx_);
}
rootFragment.verifyTree();
@@ -308,7 +309,8 @@ public class Planner {
* return a single-node, distributed, or parallel plan depending on the query and
* configuration.
*/
- public List<PlanFragment> createPlans() throws ImpalaException {
+ public List<PlanFragment> createPlans(TQueryExecRequest result)
+ throws ImpalaException {
List<PlanFragment> distrPlan = createPlanFragments();
Preconditions.checkNotNull(distrPlan);
if (useParallelPlan(ctx_)) {
@@ -318,11 +320,17 @@ public class Planner {
} else {
distrPlan = Collections.singletonList(distrPlan.get(0));
}
+
+ // Compute the processing cost and adjust fragment parallelism
+ // The TupleCachePlanner uses the processing cost and fragment parallelism, so
+ // this needs to happen first.
+ computeProcessingCost(distrPlan, result, ctx_);
+
// TupleCachePlanner comes last, because it needs to compute the eligibility of
// various locations in the PlanNode tree. Runtime filters and other modifications
// to the tree can change this, so this comes after all those modifications are
// complete.
- if (useTupleCache(ctx_)) {
+ if (useTupleCache(ctx_.getQueryOptions())) {
TupleCachePlanner cachePlanner = new TupleCachePlanner(ctx_);
distrPlan = cachePlanner.createPlans(distrPlan);
ctx_.getTimeline().markEvent("Tuple caching plan created");
@@ -345,8 +353,15 @@ public class Planner {
}
// Return true if ENABLE_TUPLE_CACHE=true
- public static boolean useTupleCache(PlannerContext planCtx) {
- return planCtx.getQueryOptions().isEnable_tuple_cache();
+ public static boolean useTupleCache(TQueryOptions queryOptions) {
+ return queryOptions.isEnable_tuple_cache();
+ }
+
+ // Return true if the processing cost has been computed. This is true for
+ // COMPUTE_PROCESSING_COST=true and ENABLE_TUPLE_CACHE=true
+ public static boolean isProcessingCostAvailable(TQueryOptions queryOptions) {
+ return queryOptions.isCompute_processing_cost() ||
+ useTupleCache(queryOptions);
}
/**
@@ -548,12 +563,11 @@ public class Planner {
* computation.
*/
public static void reduceCardinalityByRuntimeFilter(
- List<PlanFragment> planRoots, PlannerContext planCtx) {
+ PlanFragment rootFragment, PlannerContext planCtx) {
double reductionScale = planCtx.getRootAnalyzer()
.getQueryOptions()
.getRuntime_filter_cardinality_reduction_scale();
if (reductionScale <= 0) return;
- PlanFragment rootFragment = planRoots.get(0);
Stack<PlanNode> nodeStack = new Stack<>();
rootFragment.getPlanRoot().reduceCardinalityByRuntimeFilter(
nodeStack, reductionScale);
@@ -572,7 +586,7 @@ public class Planner {
List<PlanFragment> postOrderFragments = new ArrayList<>();
boolean testCostCalculation = queryOptions.isEnable_replan()
&& (RuntimeEnv.INSTANCE.isTestEnv() || queryOptions.isTest_replan());
- if (queryOptions.isCompute_processing_cost() || testCostCalculation) {
+ if (isProcessingCostAvailable(queryOptions) || testCostCalculation) {
postOrderFragments = rootFragment.getNodesPostOrder();
for (PlanFragment fragment : postOrderFragments) {
fragment.computeCostingSegment(queryOptions);
diff --git a/fe/src/main/java/org/apache/impala/planner/TupleCacheInfo.java b/fe/src/main/java/org/apache/impala/planner/TupleCacheInfo.java
index f9a6bdbcc..b3873facc 100644
--- a/fe/src/main/java/org/apache/impala/planner/TupleCacheInfo.java
+++ b/fe/src/main/java/org/apache/impala/planner/TupleCacheInfo.java
@@ -34,7 +34,9 @@ import org.apache.impala.analysis.TupleId;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.FeView;
import org.apache.impala.common.IdGenerator;
+import org.apache.impala.common.PrintUtils;
import org.apache.impala.common.ThriftSerializationCtx;
+import org.apache.impala.thrift.TExplainLevel;
import org.apache.impala.thrift.TFileSplitGeneratorSpec;
import org.apache.impala.thrift.TScanRange;
import org.apache.impala.thrift.TScanRangeLocationList;
@@ -166,6 +168,14 @@ public class TupleCacheInfo {
private boolean finalized_ = false;
private String finalizedHashString_ = null;
+ // Cumulative processing cost from all nodes that feed into this node, including nodes
+ // from other fragments (e.g. the build side of a join).
+ private long cumulativeProcessingCost_ = 0;
+
+ // Estimated size of the result at this location. This is the row size multiplied by
+ // the filtered cardinality.
+ private long estimatedSerializedSize_ = -1;
+
public TupleCacheInfo(DescriptorTable descTbl) {
ineligibilityReasons_ = EnumSet.noneOf(IneligibilityReason.class);
descriptorTable_ = descTbl;
@@ -221,6 +231,58 @@ public class TupleCacheInfo {
finalized_ = true;
}
+ public long getCumulativeProcessingCost() {
+ Preconditions.checkState(isEligible(),
+ "TupleCacheInfo only has cost information if it is cache eligible.");
+ Preconditions.checkState(finalized_, "TupleCacheInfo not finalized");
+ return cumulativeProcessingCost_;
+ }
+
+ public long getEstimatedSerializedSize() {
+ Preconditions.checkState(isEligible(),
+ "TupleCacheInfo only has cost information if it is cache eligible.");
+ Preconditions.checkState(finalized_, "TupleCacheInfo not finalized");
+ return estimatedSerializedSize_;
+ }
+
+ /**
+ * Calculate the tuple cache cost information for this plan node. This must be called
+ * with the matching PlanNode for this TupleCacheInfo. This pulls in any information
+ * from the PlanNode or from any children recursively. This cost information is used
+ * for planning decisions. It is also displayed in the explain plan output for
+ * debugging.
+ */
+ public void calculateCostInformation(PlanNode thisPlanNode) {
+ Preconditions.checkState(!finalized_,
+ "TupleCacheInfo is finalized and can't be modified");
+ Preconditions.checkState(isEligible(),
+ "TupleCacheInfo only calculates cost information if it is cache eligible.");
+ Preconditions.checkState(thisPlanNode.getTupleCacheInfo() == this,
+ "calculateCostInformation() must be called with its enclosing PlanNode");
+
+ // This was already called on our children, which are known to be eligible.
+ // Pull in the information from our children.
+ for (PlanNode child : thisPlanNode.getChildren()) {
+ cumulativeProcessingCost_ +=
+ child.getTupleCacheInfo().getCumulativeProcessingCost();
+ // If the child is from a different fragment (e.g. the build side of a hash join),
+ // incorporate the cost of the sink
+ if (child.getFragment() != thisPlanNode.getFragment()) {
+ cumulativeProcessingCost_ +=
+ child.getFragment().getSink().getProcessingCost().getTotalCost();
+ }
+ }
+ cumulativeProcessingCost_ += thisPlanNode.getProcessingCost().getTotalCost();
+
+ // If there are stats, compute the estimated serialized size. If there are no stats
+ // (i.e. cardinality == -1), then there is nothing to do.
+ if (thisPlanNode.getFilteredCardinality() > -1) {
+ long cardinality = thisPlanNode.getFilteredCardinality();
+ estimatedSerializedSize_ = (long) Math.round(
+ ExchangeNode.getAvgSerializedRowSize(thisPlanNode) * cardinality);
+ }
+ }
+
/**
* Pull in a child's TupleCacheInfo into this TupleCacheInfo. If the child is
* ineligible, then this is marked ineligible and there is no need to calculate
@@ -499,6 +561,24 @@ public class TupleCacheInfo {
return builder.toString();
}
+ /**
+ * Produce explain output describing the cost information for this tuple cache location
+ */
+ public String getCostExplainString(String detailPrefix) {
+ StringBuilder output = new StringBuilder();
+ output.append(detailPrefix + "estimated serialized size: ");
+ if (estimatedSerializedSize_ > -1) {
+ output.append(PrintUtils.printBytes(estimatedSerializedSize_));
+ } else {
+ output.append("unavailable");
+ }
+ output.append("\n");
+ output.append(detailPrefix + "cumulative processing cost: ");
+ output.append(getCumulativeProcessingCost());
+ output.append("\n");
+ return output.toString();
+ }
+
/**
* Construct a comma separated list of the ineligibility reasons.
*/
diff --git a/fe/src/main/java/org/apache/impala/planner/TupleCacheNode.java b/fe/src/main/java/org/apache/impala/planner/TupleCacheNode.java
index 98818648e..71a807ae8 100644
--- a/fe/src/main/java/org/apache/impala/planner/TupleCacheNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/TupleCacheNode.java
@@ -45,32 +45,37 @@ public class TupleCacheNode extends PlanNode {
protected boolean displayCorrectnessCheckingInfo_;
protected boolean skipCorrectnessVerification_;
protected final List<Integer> inputScanNodeIds_ = new ArrayList<Integer>();
+ protected final TupleCacheInfo childTupleCacheInfo_;
public TupleCacheNode(PlanNodeId id, PlanNode child,
boolean displayCorrectnessCheckingInfo) {
super(id, "TUPLE CACHE");
addChild(child);
cardinality_ = child.getCardinality();
+ if (child.getFilteredCardinality() != cardinality_) {
+ setFilteredCardinality(child.getFilteredCardinality());
+ }
limit_ = child.limit_;
- TupleCacheInfo childCacheInfo = child.getTupleCacheInfo();
- Preconditions.checkState(childCacheInfo.isEligible());
- compileTimeKey_ = childCacheInfo.getHashString();
+ childTupleCacheInfo_ = child.getTupleCacheInfo();
+ Preconditions.checkState(childTupleCacheInfo_.isEligible());
+ compileTimeKey_ = childTupleCacheInfo_.getHashString();
// If there is variability due to a streaming agg, skip the correctness verification
// for this location.
- skipCorrectnessVerification_ = childCacheInfo.getStreamingAggVariability();
+ skipCorrectnessVerification_ = childTupleCacheInfo_.getStreamingAggVariability();
displayCorrectnessCheckingInfo_ = displayCorrectnessCheckingInfo;
- for (HdfsScanNode scanNode : childCacheInfo.getInputScanNodes()) {
+ for (HdfsScanNode scanNode : childTupleCacheInfo_.getInputScanNodes()) {
// Inputs into the tuple cache need to use deterministic scan range assignment
scanNode.setDeterministicScanRangeAssignment(true);
inputScanNodeIds_.add(scanNode.getId().asInt());
}
+ computeTupleIds();
}
@Override
public void init(Analyzer analyzer) throws ImpalaException {
super.init(analyzer);
- computeTupleIds();
+ Preconditions.checkState(conjuncts_.isEmpty());
}
@Override
@@ -128,6 +133,7 @@ public class TupleCacheNode extends PlanNode {
inputScanNodeIds_.stream().map(Object::toString).collect(Collectors.toList());
output.append(detailPrefix + "input scan node ids: " +
String.join(",", input_scan_node_ids_strs) + "\n");
+ output.append(childTupleCacheInfo_.getCostExplainString(detailPrefix));
return output.toString();
}
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index c7f5e198f..d37e357d7 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -2031,16 +2031,14 @@ public class Frontend {
*/
private TQueryExecRequest createExecRequest(
Planner planner, PlanCtx planCtx) throws ImpalaException {
+ TQueryExecRequest result = new TQueryExecRequest();
TQueryCtx queryCtx = planner.getQueryCtx();
- List<PlanFragment> planRoots = planner.createPlans();
+ List<PlanFragment> planRoots = planner.createPlans(result);
if (planCtx.planCaptureRequested()) {
planCtx.plan_ = planRoots;
}
// Compute resource requirements of the final plans.
- TQueryExecRequest result = new TQueryExecRequest();
- Planner.reduceCardinalityByRuntimeFilter(planRoots, planner.getPlannerCtx());
- Planner.computeProcessingCost(planRoots, result, planner.getPlannerCtx());
Planner.computeResourceReqs(planRoots, queryCtx, result,
planner.getPlannerCtx(), planner.getAnalysisResult().isQueryStmt());
(impala) 01/03: IMPALA-13285: Ignore COMMIT_TXN events on Apache Hive 3
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit dc46aa48d9449729f312aaaa3312e0d8df150063
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Thu Jul 3 19:47:41 2025 +0800
IMPALA-13285: Ignore COMMIT_TXN events on Apache Hive 3
In Apache Hive 3, HMS doesn't provide the API to retrive WriteEvents
info of a given transaction. COMMIT_TXN event just contains a
transaction id so Impala can't process it.
This patch ignores COMMIT_TXN events when building on Apache Hive 3.
Some tests in MetastoreEventsProcessorTest and EventExecutorServiceTest
are skipped due to this.
Tests:
- Manually tested on Apache Hive 3. Verified that EventProcessor still
works after receiving COMMIT_TXN events.
- Passed some tests in MetastoreEventsProcessorTest and
EventExecutorServiceTest that previously failed due to EventProcessor
going into ERROR state.
Change-Id: I863e39b3702028a14e83fed1fe912b441f2c28db
Reviewed-on: http://gerrit.cloudera.org:8080/23117
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../org/apache/impala/compat/MetastoreShim.java | 32 ++++------------------
.../impala/catalog/events/MetastoreEvents.java | 5 +---
.../catalog/events/EventExecutorServiceTest.java | 8 ++++++
.../events/MetastoreEventsProcessorTest.java | 23 ++++++++++++++++
.../java/org/apache/impala/testutil/TestUtils.java | 7 +++++
5 files changed, 45 insertions(+), 30 deletions(-)
diff --git a/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
index 24fe4933b..8ed415899 100644
--- a/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
+++ b/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
@@ -74,7 +74,7 @@ import org.apache.impala.catalog.Hive3MetastoreShimBase;
import org.apache.impala.catalog.MetaStoreClientPool;
import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
import org.apache.impala.catalog.events.MetastoreEvents.DerivedMetastoreEvent;
-import org.apache.impala.catalog.events.MetastoreEvents.MetastoreEvent;
+import org.apache.impala.catalog.events.MetastoreEvents.IgnoredEvent;
import org.apache.impala.catalog.events.MetastoreEvents.MetastoreTableEvent;
import org.apache.impala.catalog.events.MetastoreEventsProcessor.MetaDataFilter;
import org.apache.impala.catalog.events.MetastoreNotificationException;
@@ -631,38 +631,17 @@ public class MetastoreShim extends Hive3MetastoreShimBase {
/**
* CDP Hive-3 only function.
*/
- public static class CommitTxnEvent extends MetastoreEvent {
+ public static class CommitTxnEvent extends IgnoredEvent {
public static final String EVENT_TYPE = "COMMIT_TXN";
public CommitTxnEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
NotificationEvent event) {
super(catalogOpExecutor, metrics, event);
- throw new UnsupportedOperationException("CommitTxnEvent is not supported.");
}
@Override
- protected void process() throws MetastoreNotificationException {
-
- }
-
- @Override
- protected boolean onFailure(Exception e) {
- return false;
- }
-
- @Override
- protected boolean isEventProcessingDisabled() {
- return false;
- }
-
- @Override
- protected SelfEventContext getSelfEventContext() {
- return null;
- }
-
- @Override
- protected boolean shouldSkipWhenSyncingToLatestEventId() {
- return false;
+ public void process() {
+ LOG.info("Ignoring COMMIT_TXN event {}", getEventId());
}
}
@@ -1045,6 +1024,7 @@ public class MetastoreShim extends Hive3MetastoreShimBase {
*/
public static List<PseudoCommitTxnEvent> getPseudoCommitTxnEvents(
CommitTxnEvent event) {
- throw new UnsupportedOperationException("CommitTxnEvent is not supported.");
+ LOG.info("Ignoring COMMIT_TXN event {}", event.getEventId());
+ return Collections.emptyList();
}
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
index 6bd4aeebc..e87985d54 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
@@ -3606,10 +3606,7 @@ public class MetastoreEvents {
*/
public static class IgnoredEvent extends MetastoreEvent {
- /**
- * Prevent instantiation from outside should use MetastoreEventFactory instead
- */
- private IgnoredEvent(
+ public IgnoredEvent(
CatalogOpExecutor catalogOpExecutor, Metrics metrics, NotificationEvent event) {
super(catalogOpExecutor, metrics, event);
}
diff --git a/fe/src/test/java/org/apache/impala/catalog/events/EventExecutorServiceTest.java b/fe/src/test/java/org/apache/impala/catalog/events/EventExecutorServiceTest.java
index 8c746ceac..793eb841c 100644
--- a/fe/src/test/java/org/apache/impala/catalog/events/EventExecutorServiceTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/events/EventExecutorServiceTest.java
@@ -53,9 +53,11 @@ import org.apache.impala.compat.MetastoreShim;
import org.apache.impala.service.BackendConfig;
import org.apache.impala.service.CatalogOpExecutor;
import org.apache.impala.testutil.CatalogServiceTestCatalog;
+import org.apache.impala.testutil.TestUtils;
import org.apache.thrift.TException;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -643,6 +645,9 @@ public class EventExecutorServiceTest {
*/
@Test
public void testCommitTxn() throws Exception {
+ Assume.assumeFalse("Skipping this since COMMIT_TXN events are ignored on Apache " +
+ "Hive 2/3. So the validWriteIds list is not updated correctly.",
+ TestUtils.isApacheHiveVersion() && TestUtils.getHiveMajorVersion() <= 3);
EventExecutorService eventExecutorService = createEventExecutorService(2, 2);
transactionTest(false);
shutDownEventExecutorService(eventExecutorService);
@@ -654,6 +659,9 @@ public class EventExecutorServiceTest {
*/
@Test
public void testAbortTxn() throws Exception {
+ Assume.assumeFalse("Skipping this since COMMIT_TXN events are ignored on Apache " +
+ "Hive 2/3. So the validWriteIds list is not updated correctly.",
+ TestUtils.isApacheHiveVersion() && TestUtils.getHiveMajorVersion() <= 3);
EventExecutorService eventExecutorService = createEventExecutorService(2, 2);
transactionTest(true);
shutDownEventExecutorService(eventExecutorService);
diff --git a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
index a409a6630..18c9f9d78 100644
--- a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
@@ -2859,6 +2859,8 @@ public class MetastoreEventsProcessorTest {
@Test
public void testCommitEvent() throws TException, ImpalaException, IOException {
+ Assume.assumeFalse("Skipping this since it depends on the behavior of CDP Hive 3",
+ TestUtils.isApacheHiveVersion());
// Turn on incremental refresh for transactional table
final TBackendGflags origCfg = BackendConfig.INSTANCE.getBackendCfg();
try {
@@ -2879,6 +2881,8 @@ public class MetastoreEventsProcessorTest {
@Test
public void testAbortEvent() throws TException, ImpalaException, IOException {
+ Assume.assumeFalse("COMMIT_TXN events are not processed on Apache Hive 2/3",
+ TestUtils.isApacheHiveVersion() && TestUtils.getHiveMajorVersion() <= 3);
// Turn on incremental refresh for transactional table
final TBackendGflags origCfg = BackendConfig.INSTANCE.getBackendCfg();
try {
@@ -3632,6 +3636,8 @@ public class MetastoreEventsProcessorTest {
*/
@Test
public void testSkipFetchOpenTransactionEvent() throws Exception {
+ Assume.assumeFalse("EventTypeSkipList is not supported on Apache Hive 2/3",
+ TestUtils.isApacheHiveVersion() && TestUtils.getHiveMajorVersion() <= 3);
long currentEventId = eventsProcessor_.getCurrentEventId();
try (MetaStoreClient client = catalog_.getMetaStoreClient()) {
// 1. Fetch notification events after open and commit transaction
@@ -3678,6 +3684,8 @@ public class MetastoreEventsProcessorTest {
*/
@Test
public void testFetchEventsInBatchWithOpenTxnAsLastEvent() throws Exception {
+ Assume.assumeFalse("EventTypeSkipList is not supported on Apache Hive 2/3",
+ TestUtils.isApacheHiveVersion() && TestUtils.getHiveMajorVersion() <= 3);
long currentEventId = eventsProcessor_.getCurrentEventId();
try (MetaStoreClient client = catalog_.getMetaStoreClient()) {
long txnId = MetastoreShim.openTransaction(client.getHiveClient());
@@ -3694,6 +3702,8 @@ public class MetastoreEventsProcessorTest {
@Test
public void testNotFetchingUnwantedEvents() throws Exception {
+ Assume.assumeFalse("EventTypeSkipList is not supported on Apache Hive 2/3",
+ TestUtils.isApacheHiveVersion() && TestUtils.getHiveMajorVersion() <= 3);
String tblName = "test_event_skip_list";
createDatabase(TEST_DB_NAME, null);
Map<String, String> params = new HashMap<>();
@@ -3840,6 +3850,8 @@ public class MetastoreEventsProcessorTest {
@Test
public void testReloadEventOnLoadedTable() throws Exception {
+ Assume.assumeFalse("RELOAD event is not generated on Apache Hive 2/3",
+ TestUtils.isApacheHiveVersion() && TestUtils.getHiveMajorVersion() <= 3);
String tblName = "test_reload";
createDatabase(TEST_DB_NAME, null);
eventsProcessor_.processEvents();
@@ -3885,6 +3897,9 @@ public class MetastoreEventsProcessorTest {
@Test
public void testCommitCompactionEventOnLoadedTable() throws Exception {
+ Assume.assumeFalse("Skipping this since COMMIT_TXN event is not supported on " +
+ "Apache Hive 2/3",
+ TestUtils.isApacheHiveVersion() && TestUtils.getHiveMajorVersion() <= 3);
String tblName = "test_commit_compaction";
createDatabase(TEST_DB_NAME, null);
eventsProcessor_.processEvents();
@@ -3984,6 +3999,8 @@ public class MetastoreEventsProcessorTest {
*/
@Test
public void testEmptyPartitionValues() throws Exception {
+ Assume.assumeFalse("RELOAD event is not generated on Apache Hive 2/3",
+ TestUtils.isApacheHiveVersion() && TestUtils.getHiveMajorVersion() <= 3);
String prevFlag = BackendConfig.INSTANCE.debugActions();
try {
String tblName = "test_empty";
@@ -4080,6 +4097,8 @@ public class MetastoreEventsProcessorTest {
public void testAllocWriteIdEvent(String tblName, boolean isPartitioned,
boolean isLoadTable) throws TException, TransactionException, CatalogException {
+ Assume.assumeFalse("COMMIT_TXN events are not processed on Apache Hive 2/3",
+ TestUtils.isApacheHiveVersion() && TestUtils.getHiveMajorVersion() <= 3);
createDatabase(TEST_DB_NAME, null);
eventsProcessor_.processEvents();
createTransactionalTable(TEST_DB_NAME, tblName, isPartitioned);
@@ -4122,6 +4141,8 @@ public class MetastoreEventsProcessorTest {
@Test
public void testNotificationEventRequest() throws Exception {
+ Assume.assumeFalse("EventTypeSkipList is not supported on Apache Hive 2/3",
+ TestUtils.isApacheHiveVersion() && TestUtils.getHiveMajorVersion() <= 3);
long currentEventId = eventsProcessor_.getCurrentEventId();
// Generate some DB only related events
createDatabaseFromImpala(TEST_DB_NAME, null);
@@ -4239,6 +4260,8 @@ public class MetastoreEventsProcessorTest {
@Test
public void testCommitTxnEventTargetName() throws Exception {
+ Assume.assumeFalse("Skipping this since it depends on the behavior of CDP Hive 3",
+ TestUtils.isApacheHiveVersion());
String tblName = "test_commit_txn";
String partTblName = "test_commit_txn_part";
String insertNonPartTbl =
diff --git a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
index 665a1b97d..b5036b37a 100644
--- a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
+++ b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
@@ -447,6 +447,13 @@ public class TestUtils {
return Integer.parseInt(hiveMajorVersion);
}
+ /**
+ * Returns whether we are using Apache Hive versions.
+ */
+ public static boolean isApacheHiveVersion() {
+ return Boolean.parseBoolean(System.getenv("USE_APACHE_HIVE"));
+ }
+
/**
* Gets checks if the catalog server running on the given host and port has
* catalog-v2 enabled