You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2026/01/26 17:18:34 UTC
(impala) branch master updated (c9bfdbb27 -> 00c233cc4)
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from c9bfdbb27 IMPALA-14682: Use centralized async cleanup for admission state cleanup
new 2360a06e4 IMPALA-14525: Calcite planner: Add support for RexSimplify
new 543f0206e IMPALA-14661: Optimize admissiond memory usage by compressing exec requests
new 03e9b63d1 IMPALA-14563: Throw AnalysisException when aggregating complex type
new 00c233cc4 IMPALA-14692: Fix test_spilling_hash_join
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/rpc/sidecar-util.h | 48 +-
be/src/rpc/thrift-util-test.cc | 32 +
be/src/rpc/thrift-util.h | 108 +-
be/src/scheduling/admission-control-service.cc | 35 +-
be/src/scheduling/admission-control-service.h | 13 +
be/src/scheduling/admission-controller-test.cc | 8 +-
be/src/scheduling/admission-controller.cc | 123 +-
be/src/scheduling/admission-controller.h | 91 +-
.../scheduling/remote-admission-control-client.cc | 20 +-
be/src/service/client-request-state.cc | 13 +-
be/src/service/client-request-state.h | 4 +
common/protobuf/admission_control_service.proto | 3 +
common/thrift/Query.thrift | 8 +
common/thrift/metrics.json | 30 +
.../apache/impala/analysis/FunctionCallExpr.java | 2 +-
.../org/apache/impala/analysis/InPredicate.java | 28 +-
.../org/apache/impala/analysis/SelectStmt.java | 16 +-
.../apache/impala/analysis/AnalyzeStmtsTest.java | 6 +
.../calcite/coercenodes/CoerceOperandShuttle.java | 60 +-
...inaryCompExpr.java => AnalyzedInPredicate.java} | 29 +-
.../impala/calcite/functions/RexCallConverter.java | 20 +
.../calcite/functions/RexLiteralConverter.java | 8 +-
.../calcite/operators/ImpalaCastFunction.java | 6 +-
.../impala/calcite/operators/ImpalaInOperator.java | 26 +-
.../impala/calcite/operators/ImpalaRexBuilder.java | 56 -
.../calcite/operators/ImpalaRexSimplify.java | 81 +
.../impala/calcite/operators/ImpalaRexUtil.java | 228 +
.../impala/calcite/rel/node/ImpalaJoinRel.java | 5 +-
.../impala/calcite/rel/util/CreateExprVisitor.java | 13 +-
.../calcite/rel/util/ExprConjunctsConverter.java | 14 +-
.../impala/calcite/rules/ImpalaCoreRules.java | 209 +
.../calcite/rules/ImpalaFilterSimplifyRule.java | 62 +
.../calcite/rules/ImpalaProjectSimplifyRule.java | 78 +
.../calcite/schema/FilterSelectivityEstimator.java | 30 +-
.../impala/calcite/service/CalciteOptimizer.java | 120 +-
.../calcite/service/CalciteRelNodeConverter.java | 32 +-
.../impala/calcite/type/ImpalaTypeConverter.java | 16 +-
.../calcite/analytic-rank-pushdown-calcite.test | 124 +-
.../calcite/limit-pushdown-analytic-calcite.test | 2 +-
.../PlannerTest/calcite_tpcds/tpcds-q01.test | 403 +-
.../PlannerTest/calcite_tpcds/tpcds-q02.test | 246 +-
.../PlannerTest/calcite_tpcds/tpcds-q03.test | 141 +-
.../PlannerTest/calcite_tpcds/tpcds-q04.test | 2087 +++---
.../PlannerTest/calcite_tpcds/tpcds-q05.test | 480 +-
.../PlannerTest/calcite_tpcds/tpcds-q06.test | 767 +--
.../PlannerTest/calcite_tpcds/tpcds-q07.test | 178 +-
.../PlannerTest/calcite_tpcds/tpcds-q08.test | 762 ++-
.../PlannerTest/calcite_tpcds/tpcds-q10a.test | 1165 ++--
.../PlannerTest/calcite_tpcds/tpcds-q11.test | 788 +--
.../PlannerTest/calcite_tpcds/tpcds-q12.test | 128 +-
.../PlannerTest/calcite_tpcds/tpcds-q13.test | 569 +-
.../PlannerTest/calcite_tpcds/tpcds-q14a.test | 6264 +++++++++---------
.../PlannerTest/calcite_tpcds/tpcds-q14b.test | 3434 +++++-----
.../PlannerTest/calcite_tpcds/tpcds-q15.test | 145 +-
.../PlannerTest/calcite_tpcds/tpcds-q16.test | 1608 ++---
.../PlannerTest/calcite_tpcds/tpcds-q17.test | 723 ++-
.../PlannerTest/calcite_tpcds/tpcds-q18.test | 347 +-
.../PlannerTest/calcite_tpcds/tpcds-q19.test | 213 +-
.../PlannerTest/calcite_tpcds/tpcds-q20.test | 146 +-
.../PlannerTest/calcite_tpcds/tpcds-q21.test | 297 +-
.../PlannerTest/calcite_tpcds/tpcds-q22.test | 59 +-
.../PlannerTest/calcite_tpcds/tpcds-q23a.test | 3586 +++++------
.../PlannerTest/calcite_tpcds/tpcds-q23b.test | 3759 +++++------
.../PlannerTest/calcite_tpcds/tpcds-q24a.test | 744 +--
.../PlannerTest/calcite_tpcds/tpcds-q24b.test | 744 +--
.../PlannerTest/calcite_tpcds/tpcds-q25.test | 719 ++-
.../PlannerTest/calcite_tpcds/tpcds-q26.test | 178 +-
.../PlannerTest/calcite_tpcds/tpcds-q27.test | 232 +-
.../PlannerTest/calcite_tpcds/tpcds-q29.test | 571 +-
.../PlannerTest/calcite_tpcds/tpcds-q30.test | 611 +-
.../PlannerTest/calcite_tpcds/tpcds-q31.test | 2602 ++++----
.../PlannerTest/calcite_tpcds/tpcds-q32.test | 661 +-
.../PlannerTest/calcite_tpcds/tpcds-q33.test | 1397 ++--
.../PlannerTest/calcite_tpcds/tpcds-q34.test | 505 +-
.../PlannerTest/calcite_tpcds/tpcds-q35a.test | 1339 ++--
.../PlannerTest/calcite_tpcds/tpcds-q36.test | 156 +-
.../PlannerTest/calcite_tpcds/tpcds-q37.test | 227 +-
.../PlannerTest/calcite_tpcds/tpcds-q38.test | 429 +-
.../PlannerTest/calcite_tpcds/tpcds-q39a.test | 128 +-
.../PlannerTest/calcite_tpcds/tpcds-q39b.test | 128 +-
.../PlannerTest/calcite_tpcds/tpcds-q40.test | 331 +-
.../PlannerTest/calcite_tpcds/tpcds-q41.test | 91 +-
.../PlannerTest/calcite_tpcds/tpcds-q42.test | 119 +-
.../calcite_tpcds/tpcds-q43-verbose.test | 143 +-
.../PlannerTest/calcite_tpcds/tpcds-q43.test | 283 +-
.../PlannerTest/calcite_tpcds/tpcds-q44.test | 202 +-
.../PlannerTest/calcite_tpcds/tpcds-q45.test | 1031 +--
.../PlannerTest/calcite_tpcds/tpcds-q46.test | 408 +-
.../PlannerTest/calcite_tpcds/tpcds-q47.test | 981 +--
.../PlannerTest/calcite_tpcds/tpcds-q48.test | 444 +-
.../PlannerTest/calcite_tpcds/tpcds-q49.test | 531 +-
.../PlannerTest/calcite_tpcds/tpcds-q50.test | 218 +-
.../PlannerTest/calcite_tpcds/tpcds-q51.test | 416 +-
.../PlannerTest/calcite_tpcds/tpcds-q52.test | 131 +-
.../PlannerTest/calcite_tpcds/tpcds-q53.test | 172 +-
.../PlannerTest/calcite_tpcds/tpcds-q54.test | 391 +-
.../PlannerTest/calcite_tpcds/tpcds-q55.test | 131 +-
.../PlannerTest/calcite_tpcds/tpcds-q56.test | 1258 ++--
.../PlannerTest/calcite_tpcds/tpcds-q57.test | 1063 ++--
.../PlannerTest/calcite_tpcds/tpcds-q58.test | 2615 ++++----
.../PlannerTest/calcite_tpcds/tpcds-q59.test | 260 +-
.../PlannerTest/calcite_tpcds/tpcds-q60.test | 1283 ++--
.../PlannerTest/calcite_tpcds/tpcds-q61.test | 435 +-
.../PlannerTest/calcite_tpcds/tpcds-q62.test | 98 +-
.../PlannerTest/calcite_tpcds/tpcds-q63.test | 180 +-
.../PlannerTest/calcite_tpcds/tpcds-q64.test | 6636 ++++++++++----------
.../PlannerTest/calcite_tpcds/tpcds-q65.test | 255 +-
.../PlannerTest/calcite_tpcds/tpcds-q66.test | 668 +-
.../PlannerTest/calcite_tpcds/tpcds-q67.test | 179 +-
.../PlannerTest/calcite_tpcds/tpcds-q68.test | 329 +-
.../PlannerTest/calcite_tpcds/tpcds-q69.test | 1721 +++--
.../PlannerTest/calcite_tpcds/tpcds-q70.test | 282 +-
.../PlannerTest/calcite_tpcds/tpcds-q71.test | 356 +-
.../PlannerTest/calcite_tpcds/tpcds-q72.test | 412 +-
.../PlannerTest/calcite_tpcds/tpcds-q73.test | 292 +-
.../PlannerTest/calcite_tpcds/tpcds-q74.test | 2128 ++++---
.../PlannerTest/calcite_tpcds/tpcds-q75.test | 1446 +++--
.../PlannerTest/calcite_tpcds/tpcds-q76.test | 480 +-
.../PlannerTest/calcite_tpcds/tpcds-q77.test | 744 ++-
.../PlannerTest/calcite_tpcds/tpcds-q78.test | 425 +-
.../PlannerTest/calcite_tpcds/tpcds-q79.test | 244 +-
.../PlannerTest/calcite_tpcds/tpcds-q80.test | 517 +-
.../PlannerTest/calcite_tpcds/tpcds-q81.test | 631 +-
.../PlannerTest/calcite_tpcds/tpcds-q82.test | 181 +-
.../PlannerTest/calcite_tpcds/tpcds-q83.test | 2393 +++----
.../PlannerTest/calcite_tpcds/tpcds-q84.test | 212 +-
.../PlannerTest/calcite_tpcds/tpcds-q85.test | 775 +--
.../PlannerTest/calcite_tpcds/tpcds-q86.test | 89 +-
.../PlannerTest/calcite_tpcds/tpcds-q87.test | 429 +-
.../PlannerTest/calcite_tpcds/tpcds-q88.test | 2770 ++++----
.../PlannerTest/calcite_tpcds/tpcds-q89.test | 254 +-
.../PlannerTest/calcite_tpcds/tpcds-q90.test | 302 +-
.../PlannerTest/calcite_tpcds/tpcds-q91.test | 286 +-
.../PlannerTest/calcite_tpcds/tpcds-q92.test | 649 +-
.../PlannerTest/calcite_tpcds/tpcds-q93.test | 412 +-
.../PlannerTest/calcite_tpcds/tpcds-q94.test | 1779 +++---
.../PlannerTest/calcite_tpcds/tpcds-q95.test | 1553 ++---
.../PlannerTest/calcite_tpcds/tpcds-q96.test | 348 +-
.../PlannerTest/calcite_tpcds/tpcds-q97.test | 248 +-
.../PlannerTest/calcite_tpcds/tpcds-q98.test | 146 +-
.../PlannerTest/calcite_tpcds/tpcds-q99.test | 98 +-
.../queries/QueryTest/query-impala-13138.test | 2 +-
tests/custom_cluster/test_admission_controller.py | 6 +
143 files changed, 44997 insertions(+), 40368 deletions(-)
copy java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/{AnalyzedBinaryCompExpr.java => AnalyzedInPredicate.java} (69%)
copy fe/src/main/java/org/apache/impala/analysis/SystemTableRef.java => java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaInOperator.java (55%)
delete mode 100644 java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaRexBuilder.java
create mode 100644 java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaRexSimplify.java
create mode 100644 java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaRexUtil.java
create mode 100644 java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaCoreRules.java
create mode 100644 java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaFilterSimplifyRule.java
create mode 100644 java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaProjectSimplifyRule.java
(impala) 03/04: IMPALA-14563: Throw AnalysisException when aggregating complex type
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 03e9b63d166c402b1e5a2b55123bbcc84cb48ddc
Author: Surya Hebbar <sh...@cloudera.com>
AuthorDate: Mon Jan 12 15:17:50 2026 +0530
IMPALA-14563: Throw AnalysisException when aggregating complex type
Before this change, when the user tries to aggregate by a complex type,
we threw an IllegalStateException.
SELECT complex_col, count(*) FROM table_example GROUP BY complex_col;
Error:
java.lang.IllegalStateException: null
This does not provide appropriate context about the query's problem.
Instead of throwing an IllegalStateException, we now throw the
appropriate AnalysisException, which highlights the issue with the query.
AnalysisException: GROUP BY expression cannot be used on complex types
without specifying a field
This helps the user rewrite the query correctly by making them aware that
specifying a field would fix the query.
SELECT complex_col.val1, count(*) FROM table_example GROUP BY complex_col.val1;
Testing:
Added tests to TestsGroupBy() method in AnalyzeStmtsTest.java
Change-Id: Ie431873efe7100e8150e93be2aa016381d687268
Reviewed-on: http://gerrit.cloudera.org:8080/23855
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Noemi Pap-Takacs <np...@cloudera.com>
---
.../main/java/org/apache/impala/analysis/SelectStmt.java | 16 ++++++++++------
.../org/apache/impala/analysis/AnalyzeStmtsTest.java | 6 ++++++
2 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
index 03c3f8e23..da7cef4a1 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
@@ -1149,18 +1149,22 @@ public class SelectStmt extends QueryStmt {
substituteOrdinalsAndAliases(groupingExprsCopy_, "GROUP BY", analyzer_);
for (int i = 0; i < groupingExprsCopy_.size(); ++i) {
- groupingExprsCopy_.get(i).analyze(analyzer_);
- if (groupingExprsCopy_.get(i).contains(Expr.IS_AGGREGATE)) {
- // reference the original expr in the error msg
+ Expr expr = groupingExprsCopy_.get(i);
+ expr.analyze(analyzer_);
+ if (expr.contains(Expr.IS_AGGREGATE)) {
throw new AnalysisException(
"GROUP BY expression must not contain aggregate functions: "
+ groupingExprs_.get(i).toSql());
}
- if (groupingExprsCopy_.get(i).contains(AnalyticExpr.class)) {
- // reference the original expr in the error msg
+ if (expr.contains(AnalyticExpr.class)) {
throw new AnalysisException(
"GROUP BY expression must not contain analytic expressions: "
- + groupingExprsCopy_.get(i).toSql());
+ + expr.toSql());
+ }
+ if (expr.getType().isComplexType()) {
+ throw new AnalysisException(
+ "GROUP BY expression cannot be used on complex types without specifying a "
+ + "field: " + expr.toSql());
}
}
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
index 91e3cb2cd..961f90372 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
@@ -3001,6 +3001,12 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
AnalysisError("select zip, count(*) from functional.testtbl group by 2",
"GROUP BY expression must not contain aggregate functions");
+ // can't group a complex type
+ AnalysisError("select int_struct_col, count(*) from "
+ + "functional_parquet.allcomplextypes" + " group by int_struct_col",
+ "GROUP BY expression cannot be used on complex types without specifying a field:"
+ + " int_struct_col");
+
// multiple grouping cols
AnalyzesOk("select int_col, string_col, bigint_col, count(*) " +
"from functional.alltypes group by string_col, int_col, bigint_col");
(impala) 02/04: IMPALA-14661: Optimize admissiond memory usage by compressing exec requests
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 543f0206eb68429c5bc7d69f2623addf71f57f7b
Author: Yida Wu <yi...@cloudera.com>
AuthorDate: Thu Dec 18 02:28:47 2025 -0800
IMPALA-14661: Optimize admissiond memory usage by compressing exec requests
In global admissiond, the TQueryExecRequest can be very large
for complex queries, consuming large memory while queries are
queued.
This patch adds support for compressing TQueryExecRequest when
sending it to the admission control service through AdmitQuery RPC.
This reduces memory usage in admissiond for large query execution
requests. Compression is controlled by the new startup flag
admission_control_rpc_compress_threshold_bytes. A value of 0
disables compression, while positive values enable compression for
requests larger than the threshold. The uncompressed path remains
unchanged.
Adds a new TQueryExecRequestCompressed thrift struct along with
compression and decompression helper functions. The admission
controller now handles both compressed and uncompressed requests
through a common AdmissionExecRequest abstraction.
Compressed requests are decompressed lazily and cached to reduce
decompression overhead. Decompression timing is carefully controlled.
Requests are initially decompressed at submission, but if a request
is queued, the decompressed request cache is released to reduce
memory usage.
When a queued request is later dequeued, it is decompressed again
and the decompressed cache is retained. Since admission uses FIFO
ordering, a dequeued request is expected to be at the head of the
queue and may be accessed multiple times if not admitted
immediately. Retaining the cache in this case avoids repeated
decompression.
This patch also removes the query_options field in AdmissionRequest
to eliminate ambiguity between TExecRequest.query_options and the
query options nested in TQueryExecRequest. ClientRequestState is
updated to sync the top-level TExecRequest.query_options into the
nested TQueryExecRequest before admission. As a result, the
admission controller now reads query options from TQueryExecRequest,
enforcing a single source of truth for admission logic.
Adds admissiond metrics to track compressed size, uncompressed size,
and compression ratio for query execution requests.
Testing:
Adds unit tests for Thrift compression and decompression helpers.
Updates admission controller tests to cover compressed requests.
Passed exhaustive tests.
Change-Id: I5a676d1a806451cbf84b0a3f8a706d7c6655e12d
Reviewed-on: http://gerrit.cloudera.org:8080/23852
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Yida Wu <wy...@gmail.com>
---
be/src/rpc/sidecar-util.h | 48 ++++++--
be/src/rpc/thrift-util-test.cc | 32 ++++++
be/src/rpc/thrift-util.h | 108 +++++++++++++++++-
be/src/scheduling/admission-control-service.cc | 35 ++++--
be/src/scheduling/admission-control-service.h | 13 +++
be/src/scheduling/admission-controller-test.cc | 8 +-
be/src/scheduling/admission-controller.cc | 123 ++++++++++++++++++---
be/src/scheduling/admission-controller.h | 91 ++++++++++++++-
.../scheduling/remote-admission-control-client.cc | 20 ++--
be/src/service/client-request-state.cc | 13 +--
be/src/service/client-request-state.h | 4 +
common/protobuf/admission_control_service.proto | 3 +
common/thrift/Query.thrift | 8 ++
common/thrift/metrics.json | 30 +++++
tests/custom_cluster/test_admission_controller.py | 6 +
15 files changed, 483 insertions(+), 59 deletions(-)
diff --git a/be/src/rpc/sidecar-util.h b/be/src/rpc/sidecar-util.h
index d1d7956b3..35f022afe 100644
--- a/be/src/rpc/sidecar-util.h
+++ b/be/src/rpc/sidecar-util.h
@@ -31,27 +31,53 @@ namespace impala {
class KrpcSerializer {
public:
- KrpcSerializer() : serializer_(/* compact */ true) {}
+ KrpcSerializer(int64_t threshold_bytes = 0)
+ : serializer_(/* compact */ true), compress_threshold_bytes_(threshold_bytes) {}
/// Serialize obj and set it as a sidecar on 'rpc_controller', returning the idx in
- /// 'sidecar_idx'. The memory for the sidecar is owned by this object and must remain
- /// valid until the rpc has completed.
- template <class T>
- Status SerializeToSidecar(
- const T* obj, kudu::rpc::RpcController* rpc_controller, int* sidecar_idx) {
- uint8_t* serialized_buf = nullptr;
- uint32_t serialized_len = 0;
- RETURN_IF_ERROR(serializer_.SerializeToBuffer(obj, &serialized_len, &serialized_buf));
+ /// 'sidecar_idx'.
+ ///
+ /// If TCompressed is provided and the serialized size exceeds
+ /// 'compress_threshold_bytes_', the object is compressed and the TCompressed object
+ /// is sent instead. Compression is disabled when the threshold is 0.
+ ///
+ /// The sidecar memory is owned by this serializer and must remain valid until the rpc
+ /// has completed. If provided, 'is_compressed' indicates whether the sidecar thrift
+ /// object is compressed.
+ template <typename TCompressed = void, typename T>
+ Status SerializeToSidecar(const T* obj, kudu::rpc::RpcController* rpc_controller,
+ int* sidecar_idx, bool* is_compressed = nullptr) {
+ uint8_t* buf = nullptr;
+ uint32_t len = 0;
+ RETURN_IF_ERROR(serializer_.SerializeToBuffer(obj, &len, &buf));
+
+ if (is_compressed != nullptr) *is_compressed = false;
+
+ if constexpr (!std::is_same_v<TCompressed, void>) {
+ // Compression path.
+ if (compress_threshold_bytes_ > 0 && len > compress_threshold_bytes_) {
+ // NOTE: 'buf' points to the serializer's internal buffer.
+ // Ensure 'buf' is fully consumed into a compressed thrift in
+ // CreateCompressedThrift() before the next call to SerializeToBuffer()
+ // that could reset or reuse the serializer buffer.
+ TCompressed compressed_obj;
+ RETURN_IF_ERROR(CreateCompressedThrift<TCompressed>(buf, len, &compressed_obj));
+ RETURN_IF_ERROR(serializer_.SerializeToBuffer(&compressed_obj, &len, &buf));
+ if (is_compressed != nullptr) *is_compressed = true;
+ }
+ }
+
std::unique_ptr<kudu::rpc::RpcSidecar> rpc_sidecar =
- kudu::rpc::RpcSidecar::FromSlice(kudu::Slice(serialized_buf, serialized_len));
+ kudu::rpc::RpcSidecar::FromSlice(kudu::Slice(buf, len));
KUDU_RETURN_IF_ERROR(
- rpc_controller->AddOutboundSidecar(move(rpc_sidecar), sidecar_idx),
+ rpc_controller->AddOutboundSidecar(std::move(rpc_sidecar), sidecar_idx),
"Failed to add sidecar");
return Status::OK();
}
private:
ThriftSerializer serializer_;
+ int64_t compress_threshold_bytes_;
};
// Retrieves the sidecar at 'sidecar_idx' from 'rpc_context' and deserializes it into
diff --git a/be/src/rpc/thrift-util-test.cc b/be/src/rpc/thrift-util-test.cc
index 427cb3d42..5b90cf241 100644
--- a/be/src/rpc/thrift-util-test.cc
+++ b/be/src/rpc/thrift-util-test.cc
@@ -129,4 +129,36 @@ TEST(ThriftUtil, SerDeBuffer100MB) {
}
}
+struct MockCompressedThrift {
+ std::string compressed_data;
+ int64_t uncompressed_size;
+};
+
+TEST(ThriftUtil, CompressDecompressQueryOptions) {
+ TQueryOptions options;
+ options.__set_mem_limit(1024 * 1024 * 1024LL);
+ options.__set_query_timeout_s(300);
+ options.__set_abort_on_error(true);
+
+ // Compress TQueryOptions.
+ MockCompressedThrift compressed_obj;
+ Status status =
+ CompressThrift<TQueryOptions, MockCompressedThrift>(options, &compressed_obj);
+
+ EXPECT_TRUE(status.ok()) << status.GetDetail();
+ EXPECT_GT(compressed_obj.uncompressed_size, 0);
+ EXPECT_GT(compressed_obj.compressed_data.size(), 0);
+
+ // Decompress TQueryOptions.
+ TQueryOptions decompressed_options;
+ status = DecompressThrift<MockCompressedThrift, TQueryOptions>(
+ compressed_obj, &decompressed_options);
+
+ EXPECT_TRUE(status.ok()) << status.GetDetail();
+
+ // Verify the fields.
+ EXPECT_EQ(options.mem_limit, decompressed_options.mem_limit);
+ EXPECT_EQ(options.query_timeout_s, decompressed_options.query_timeout_s);
+ EXPECT_EQ(options.abort_on_error, decompressed_options.abort_on_error);
+}
}
diff --git a/be/src/rpc/thrift-util.h b/be/src/rpc/thrift-util.h
index 311a636a6..fbb77a46f 100644
--- a/be/src/rpc/thrift-util.h
+++ b/be/src/rpc/thrift-util.h
@@ -19,18 +19,20 @@
#ifndef IMPALA_RPC_THRIFT_UTIL_H
#define IMPALA_RPC_THRIFT_UTIL_H
-#include <boost/shared_ptr.hpp>
-#include <thrift/protocol/TBinaryProtocol.h>
#include <sstream>
#include <vector>
+
+#include <boost/shared_ptr.hpp>
#include <thrift/TApplicationException.h>
#include <thrift/TConfiguration.h>
+#include <thrift/protocol/TBinaryProtocol.h>
#include <thrift/protocol/TDebugProtocol.h>
#include <thrift/transport/TBufferTransports.h>
#include <thrift/transport/TSSLSocket.h>
#include <thrift/transport/TTransportException.h>
#include "common/status.h"
+#include "util/codec.h"
namespace impala {
@@ -267,6 +269,108 @@ bool IsPeekTimeoutTException(const apache::thrift::transport::TTransportExceptio
/// Returns true if the exception indicates the other end of the TCP socket was closed.
bool IsConnResetTException(const apache::thrift::transport::TTransportException& e);
+
+// Helper that compresses a raw buffer using LZ4 and creates a TCompressed object.
+//
+// TCompressed must have:
+// - int64_t uncompressed_size;
+// - binary compressed_data;
+//
+// Params:
+// serialized_buf: Raw uncompressed data to compress.
+// serialized_len: Length of the uncompressed data in bytes.
+// dest: Output parameter for the compressed thrift object.
+template <typename TCompressed>
+Status CreateCompressedThrift(
+ const uint8_t* serialized_buf, uint32_t serialized_len, TCompressed* dest) {
+ DCHECK(serialized_buf != nullptr);
+ DCHECK(dest != nullptr);
+
+ // Prepare compressor.
+ boost::scoped_ptr<Codec> compressor;
+ Codec::CodecInfo codec_info(THdfsCompression::LZ4);
+ RETURN_IF_ERROR(Codec::CreateCompressor(nullptr, false, codec_info, &compressor));
+
+ // Calculate buffer size and resize string.
+ int64_t max_out = compressor->MaxOutputLen(serialized_len);
+ std::string compressed_str(max_out, '\0');
+
+ // Perform Compression.
+ uint8_t* out_ptr = reinterpret_cast<uint8_t*>(compressed_str.data());
+ int64_t actual_len = max_out;
+ RETURN_IF_ERROR(compressor->ProcessBlock(
+ true, serialized_len, serialized_buf, &actual_len, &out_ptr));
+ // Validate output length.
+ if (actual_len <= 0 || actual_len > std::numeric_limits<uint32_t>::max()) {
+ return Status(strings::Substitute(
+ "Invalid actual compressed length in compressed request: $0B", actual_len));
+ }
+ compressed_str.resize(actual_len);
+
+ // Write to the TCompressed object.
+ dest->uncompressed_size = static_cast<int64_t>(serialized_len);
+ dest->compressed_data = std::move(compressed_str);
+
+ return Status::OK();
+}
+
+// Serializes a T object 'src', creates a compressed thrift by the serialized bytes,
+// and writes to a compressed T object 'dest'.
+//
+// Params:
+// src: The thrift object to serialize and compress.
+// dest: Output parameter for the compressed thrift object.
+template <typename T, typename TCompressed>
+Status CompressThrift(const T& src, TCompressed* dest) {
+ ThriftSerializer serializer(/* compact */ true);
+ uint8_t* serialized_buf = nullptr;
+ uint32_t serialized_len = 0;
+ RETURN_IF_ERROR(serializer.SerializeToBuffer(&src, &serialized_len, &serialized_buf));
+ return CreateCompressedThrift<TCompressed>(serialized_buf, serialized_len, dest);
+}
+
+// Decompresses a compressed Thrift object 'src' and deserializes it back into
+// the standard Thrift object 'dest'.
+//
+// TCompressed must have:
+// - int64_t uncompressed_size;
+// - binary compressed_data;
+//
+// Params:
+// src: The compressed thrift object to decompress and deserialize.
+// dest: Output parameter for the deserialized thrift object.
+template <typename TCompressed, typename T>
+Status DecompressThrift(const TCompressed& src, T* dest) {
+ DCHECK(dest != nullptr);
+ int64_t uncompressed_len = src.uncompressed_size;
+ if (uncompressed_len <= 0 || uncompressed_len > std::numeric_limits<uint32_t>::max()) {
+ return Status(strings::Substitute(
+ "Invalid uncompressed size in compressed request: $0B", uncompressed_len));
+ }
+
+ // Prepare decompressor and decompress.
+ boost::scoped_ptr<Codec> decompressor;
+ RETURN_IF_ERROR(
+ Codec::CreateDecompressor(nullptr, false, THdfsCompression::LZ4, &decompressor));
+ std::vector<uint8_t> uncompressed_buf(uncompressed_len);
+ uint8_t* out_ptr = uncompressed_buf.data();
+ int64_t actual_out = uncompressed_len;
+ const std::string& input_data = src.compressed_data;
+ const uint8_t* in_ptr = reinterpret_cast<const uint8_t*>(input_data.data());
+ RETURN_IF_ERROR(
+ decompressor->ProcessBlock(true, input_data.size(), in_ptr, &actual_out, &out_ptr));
+ if (actual_out <= 0 || actual_out > std::numeric_limits<uint32_t>::max()) {
+ return Status(strings::Substitute(
+ "Invalid actual output size after decompression: $0B", actual_out));
+ }
+ if (actual_out != uncompressed_len) {
+ return Status("Decompressed size did not match expected size from header");
+ }
+
+ // Deserialize and write to the T object.
+ uint32_t deser_len = static_cast<uint32_t>(actual_out);
+ return DeserializeThriftMsg(uncompressed_buf.data(), &deser_len, true, dest);
+}
}
#endif
diff --git a/be/src/scheduling/admission-control-service.cc b/be/src/scheduling/admission-control-service.cc
index c5b7aaa9b..f57287e69 100644
--- a/be/src/scheduling/admission-control-service.cc
+++ b/be/src/scheduling/admission-control-service.cc
@@ -151,16 +151,21 @@ void AdmissionControlService::AdmitQuery(
VLOG(1) << "AdmitQuery: query_id=" << req->query_id()
<< " coordinator=" << req->coord_id();
- shared_ptr<AdmissionState> admission_state;
- admission_state = make_shared<AdmissionState>(req->query_id(), req->coord_id());
- admission_state->query_exec_request = make_unique<TQueryExecRequest>();
-
+ shared_ptr<AdmissionState> admission_state =
+ make_shared<AdmissionState>(req->query_id(), req->coord_id());
admission_state->summary_profile =
RuntimeProfile::Create(&admission_state->profile_pool, "Summary");
+ if (req->has_is_compressed() && req->is_compressed()) {
+ admission_state->query_exec_request_compressed =
+ make_unique<TQueryExecRequestCompressed>();
+ RESPOND_IF_ERROR(GetSidecar(req->query_exec_request_sidecar_idx(), rpc_context,
+ admission_state->query_exec_request_compressed.get()));
- RESPOND_IF_ERROR(GetSidecar(req->query_exec_request_sidecar_idx(), rpc_context,
- admission_state->query_exec_request.get()));
-
+ } else {
+ admission_state->query_exec_request = make_unique<TQueryExecRequest>();
+ RESPOND_IF_ERROR(GetSidecar(req->query_exec_request_sidecar_idx(), rpc_context,
+ admission_state->query_exec_request.get()));
+ }
for (const NetworkAddressPB& address : req->blacklisted_executor_addresses()) {
admission_state->blacklisted_executor_addresses.emplace(address);
}
@@ -212,7 +217,7 @@ void AdmissionControlService::GetQueryStatus(const GetQueryStatusRequestPB* req,
if (admission_state->admit_status.ok()) {
*resp->mutable_query_schedule() = *admission_state->schedule.get();
// Free TQueryExecRequest since it's not required after admission is done
- admission_state->query_exec_request.reset();
+ admission_state->ReleaseQueryExecRequest();
} else {
status = admission_state->admit_status;
}
@@ -359,9 +364,19 @@ void AdmissionControlService::AdmitFromThreadPool(const UniqueIdPB& query_id) {
{
lock_guard<mutex> l(admission_state->lock);
bool queued;
+ if (admission_state->query_exec_request) {
+ admission_state->admission_exec_request =
+ std::make_unique<AdmissionExecRequestUncompressed>(
+ admission_state->query_exec_request.get());
+ } else {
+ admission_state->admission_exec_request =
+ std::make_unique<AdmissionExecRequestCompressed>(
+ admission_state->query_exec_request_compressed.get(),
+ AdmissiondEnv::GetInstance()->admission_controller());
+ }
+
AdmissionController::AdmissionRequest request = {admission_state->query_id,
- admission_state->coord_id, *admission_state->query_exec_request,
- admission_state->query_exec_request->query_ctx.client_request.query_options,
+ admission_state->coord_id, *admission_state->admission_exec_request,
admission_state->summary_profile,
admission_state->blacklisted_executor_addresses};
admission_state->admit_status =
diff --git a/be/src/scheduling/admission-control-service.h b/be/src/scheduling/admission-control-service.h
index 762bc49a2..365f48fb3 100644
--- a/be/src/scheduling/admission-control-service.h
+++ b/be/src/scheduling/admission-control-service.h
@@ -92,11 +92,24 @@ class AdmissionControlService : public AdmissionControlServiceIf,
AdmissionState(const UniqueIdPB& query_id, const UniqueIdPB& coord_id)
: query_id(query_id), coord_id(coord_id) {}
+ ~AdmissionState() { ReleaseQueryExecRequest(); }
+
+ void ReleaseQueryExecRequest() {
+ // 'admission_exec_request' holds the raw pointer to the request objects,
+ // so ensure it is destroyed before the requests become invalid.
+ admission_exec_request.reset();
+ query_exec_request.reset();
+ query_exec_request_compressed.reset();
+ }
+
// The following are copied from the AdmitQueryRequestPB for this query and are valid
// at any point after this AdmissionState has been added to 'admission_state_map_'.
UniqueIdPB query_id;
UniqueIdPB coord_id;
std::unique_ptr<TQueryExecRequest> query_exec_request;
+ std::unique_ptr<TQueryExecRequestCompressed> query_exec_request_compressed;
+ std::unique_ptr<AdmissionExecRequest> admission_exec_request;
+
std::unordered_set<NetworkAddressPB> blacklisted_executor_addresses;
// Protects all of the following members.
diff --git a/be/src/scheduling/admission-controller-test.cc b/be/src/scheduling/admission-controller-test.cc
index 8dcdbc569..694e6d341 100644
--- a/be/src/scheduling/admission-controller-test.cc
+++ b/be/src/scheduling/admission-controller-test.cc
@@ -415,10 +415,12 @@ class AdmissionControllerTest : public testing::Test {
UniqueIdPB* query_id = pool_.Add(new UniqueIdPB());
RuntimeProfile* summary_profile = pool_.Add(new RuntimeProfile(&pool_, "foo"));
std::unordered_set<NetworkAddressPB> blacklisted_executor_addresses;
- const TQueryExecRequest& exec_request = schedule_state->request();
+ const TQueryExecRequest& t_exec_request = schedule_state->request();
+ AdmissionExecRequestUncompressed* exec_request =
+ pool_.Add(new AdmissionExecRequestUncompressed(&t_exec_request));
TQueryOptions query_options;
- AdmissionController::AdmissionRequest request = {*query_id, *coord_id, exec_request,
- query_options, summary_profile, blacklisted_executor_addresses};
+ AdmissionController::AdmissionRequest request = {*query_id, *coord_id, *exec_request,
+ summary_profile, blacklisted_executor_addresses};
// Clear queue_nodes_ so we can call this method again, though this means there can
// only ever be one queue node.
diff --git a/be/src/scheduling/admission-controller.cc b/be/src/scheduling/admission-controller.cc
index 75b9ea3ec..3f9f1c1e7 100644
--- a/be/src/scheduling/admission-controller.cc
+++ b/be/src/scheduling/admission-controller.cc
@@ -38,6 +38,7 @@
#include "util/bit-util.h"
#include "util/collection-metrics.h"
#include "util/debug-util.h"
+#include "util/histogram-metric.h"
#include "util/memory-metrics.h"
#include "util/metrics.h"
#include "util/pretty-printer.h"
@@ -127,6 +128,12 @@ const string EXEC_GROUP_QUERY_LOAD_KEY_FORMAT =
const string TOTAL_DEQUEUE_FAILED_COORDINATOR_LIMITED =
"admission-controller.total-dequeue-failed-coordinator-limited";
+const string EXEC_REQ_COMPRESSED_SIZE_KEY =
+ "admission-controller.query-exec-request-compressed-bytes";
+const string EXEC_REQ_UNCOMPRESSED_SIZE_KEY =
+ "admission-controller.query-exec-request-uncompressed-bytes";
+const string EXEC_REQ_COMPRESSION_RATIO_KEY =
+ "admission-controller.query-exec-request-compression-ratio";
// Define metric key format strings for metrics in PoolMetrics
// '$0' is replaced with the pool name by strings::Substitute
@@ -346,6 +353,55 @@ static inline bool ParsePoolTopicKey(
return true;
}
+Status AdmissionExecRequestCompressed::GetQueryExecRequest(
+ const TQueryExecRequest** out_req) const {
+ lock_guard<std::mutex> l(lock_);
+ if (decompressed_req_ != nullptr) {
+ *out_req = decompressed_req_.get();
+ return Status::OK();
+ }
+ DCHECK(req_ != nullptr);
+
+ unique_ptr<TQueryExecRequest> new_req = std::make_unique<TQueryExecRequest>();
+ RETURN_IF_ERROR(DecompressThrift(*req_, new_req.get()));
+
+ int64_t comp_size = req_->compressed_data.size();
+ int64_t uncomp_size = req_->uncompressed_size;
+ float ratio = (comp_size > 0) ? (static_cast<float>(uncomp_size) / comp_size) : 0.0f;
+
+ LOG(INFO) << "Decompress TQueryExecRequest for query "
+ << PrintId(new_req->query_ctx.query_id) << ": Compressed size=" << comp_size
+ << " B"
+ << ", Uncompressed size=" << uncomp_size << " B"
+ << ", Ratio=" << ratio << "x";
+
+ // Update the compression stats only once.
+ if (first_decompress_ && admission_controller_ != nullptr) {
+ admission_controller_->UpdateAdmissionRequestCompressionStats(
+ comp_size, uncomp_size, ratio);
+ }
+
+ // Store the object in our cache.
+ decompressed_req_ = std::move(new_req);
+ *out_req = decompressed_req_.get();
+ first_decompress_ = false;
+
+ return Status::OK();
+}
+
+void AdmissionController::UpdateAdmissionRequestCompressionStats(
+ int64_t compressed_size, int64_t uncompressed_size, float ratio) const {
+ if (compressed_size_metric_ != nullptr) {
+ compressed_size_metric_->Update(compressed_size);
+ }
+ if (uncompressed_size_metric_ != nullptr) {
+ uncompressed_size_metric_->Update(uncompressed_size);
+ }
+ if (compression_ratio_metric_ != nullptr) {
+ compression_ratio_metric_->Update(ratio);
+ }
+}
+
// Append to ss a debug string for memory consumption part of the pool stats.
// Here is one example.
// topN_query_stats: queries=[554b016cf0f3a37f:9a1bfcfd00000000,
@@ -700,6 +756,13 @@ AdmissionController::AdmissionController(ClusterMembershipMgr* cluster_membershi
});
total_dequeue_failed_coordinator_limited_ =
metrics_group_->AddCounter(TOTAL_DEQUEUE_FAILED_COORDINATOR_LIMITED, 0);
+ compressed_size_metric_ = metrics_group_->RegisterMetric(new HistogramMetric(
+ MetricDefs::Get(EXEC_REQ_COMPRESSED_SIZE_KEY), numeric_limits<int64_t>::max(), 3));
+ uncompressed_size_metric_ = metrics_group_->RegisterMetric(
+ new HistogramMetric(MetricDefs::Get(EXEC_REQ_UNCOMPRESSED_SIZE_KEY),
+ numeric_limits<int64_t>::max(), 3));
+ compression_ratio_metric_ = metrics_group_->RegisterMetric(
+ new HistogramMetric(MetricDefs::Get(EXEC_REQ_COMPRESSION_RATIO_KEY), 10000, 3));
if (FLAGS_cluster_membership_topic_id.empty()) {
request_queue_topic_name_ = Statestore::IMPALA_REQUEST_QUEUE_TOPIC;
} else {
@@ -1584,7 +1647,11 @@ Status AdmissionController::SubmitForAdmission(const AdmissionRequest& request,
unique_ptr<QuerySchedulePB>* schedule_result, bool& queued,
std::string* request_pool) {
queued = false;
- DebugActionNoFail(request.query_options, "AC_BEFORE_ADMISSION");
+ const TQueryExecRequest* exec_req = nullptr;
+ RETURN_IF_ERROR(request.request.GetQueryExecRequest(&exec_req));
+ DCHECK(exec_req != nullptr);
+ DebugActionNoFail(
+ exec_req->query_ctx.client_request.query_options, "AC_BEFORE_ADMISSION");
DCHECK(schedule_result->get() == nullptr);
ClusterMembershipMgr::SnapshotPtr membership_snapshot =
@@ -1618,8 +1685,8 @@ Status AdmissionController::SubmitForAdmission(const AdmissionRequest& request,
// Re-resolve the pool name to propagate any resolution errors now that this request is
// known to require a valid pool. All executor groups / schedules will use the same pool
// name.
- RETURN_IF_ERROR(ResolvePoolAndGetConfig(request.request.query_ctx,
- &queue_node->pool_name, &queue_node->pool_cfg, &queue_node->root_cfg));
+ RETURN_IF_ERROR(ResolvePoolAndGetConfig(exec_req->query_ctx, &queue_node->pool_name,
+ &queue_node->pool_cfg, &queue_node->root_cfg));
request.summary_profile->AddInfoString("Request Pool", queue_node->pool_name);
{
@@ -1678,8 +1745,11 @@ Status AdmissionController::SubmitForAdmission(const AdmissionRequest& request,
}
string user;
- RETURN_IF_ERROR(GetEffectiveShortUser(
- queue_node->admission_request.request.query_ctx.session, &user));
+ const TQueryExecRequest* exec_req_queue_node = nullptr;
+ RETURN_IF_ERROR(
+ queue_node->admission_request.request.GetQueryExecRequest(&exec_req_queue_node));
+ DCHECK(exec_req_queue_node != nullptr);
+ RETURN_IF_ERROR(GetEffectiveShortUser(exec_req_queue_node->query_ctx.session, &user));
if (queue_node->admitted_schedule.get() != nullptr) {
DCHECK(queue_node->admitted_schedule->query_schedule_pb().get() != nullptr);
@@ -1719,6 +1789,8 @@ Status AdmissionController::SubmitForAdmission(const AdmissionRequest& request,
stats->IncrementPerUser(user);
}
queue->Enqueue(queue_node);
+ // Clear the decompressed cache to save the memory after enqueue.
+ queue_node->admission_request.request.ClearDecompressedCache();
// Must be done while we still hold 'admission_ctrl_lock_' as the dequeue loop thread
// can modify 'not_admitted_reason'.
@@ -1786,9 +1858,13 @@ Status AdmissionController::WaitOnQueued(const UniqueIdPB& query_id,
queue_nodes_.erase(query_id);
});
+ const TQueryExecRequest* exec_req_queue_node = nullptr;
+ RETURN_IF_ERROR(
+ queue_node->admission_request.request.GetQueryExecRequest(&exec_req_queue_node));
+ DCHECK(exec_req_queue_node != nullptr);
// Disallow the FAIL action here. It would leave the queue in an inconsistent state.
- DebugActionNoFail(
- queue_node->admission_request.query_options, "AC_AFTER_ADMISSION_OUTCOME");
+ DebugActionNoFail(exec_req_queue_node->query_ctx.client_request.query_options,
+ "AC_AFTER_ADMISSION_OUTCOME");
{
lock_guard<mutex> lock(admission_ctrl_lock_);
@@ -2270,6 +2346,9 @@ Status AdmissionController::ComputeGroupScheduleStates(
return Status::OK();
}
const AdmissionRequest& request = queue_node->admission_request;
+ const TQueryExecRequest* exec_req = nullptr;
+ RETURN_IF_ERROR(request.request.GetQueryExecRequest(&exec_req));
+ DCHECK(exec_req != nullptr);
VLOG(3) << "Scheduling query " << PrintId(request.query_id)
<< " with membership version " << current_membership_version;
@@ -2304,7 +2383,7 @@ Status AdmissionController::ComputeGroupScheduleStates(
executor_groups = {&membership_snapshot->all_coordinators};
} else {
executor_groups =
- GetExecutorGroupsForQuery(membership_snapshot->executor_groups, request);
+ GetExecutorGroupsForQuery(membership_snapshot->executor_groups, *exec_req);
}
if (executor_groups.empty()) {
@@ -2314,8 +2393,9 @@ Status AdmissionController::ComputeGroupScheduleStates(
}
// Collect all coordinators if needed for the request.
- ExecutorGroup coords = request.request.include_all_coordinators ?
- membership_snapshot->all_coordinators : ExecutorGroup("all-coordinators");
+ ExecutorGroup coords = exec_req->include_all_coordinators ?
+ membership_snapshot->all_coordinators :
+ ExecutorGroup("all-coordinators");
// We loop over the executor groups in a deterministic order. If
// --balance_queries_across_executor_groups set to true, executor groups with more
@@ -2342,7 +2422,8 @@ Status AdmissionController::ComputeGroupScheduleStates(
}
unique_ptr<ScheduleState> group_state = make_unique<ScheduleState>(request.query_id,
- request.request, request.query_options, request.summary_profile, false);
+ *exec_req, exec_req->query_ctx.client_request.query_options,
+ request.summary_profile, false);
const string& group_name = executor_group->name();
VLOG(3) << "Scheduling for executor group: " << group_name << " with "
<< executor_group->NumExecutors() << " executors";
@@ -2628,6 +2709,17 @@ void AdmissionController::TryDequeue() {
VLOG(3) << "Dequeueing from stats for pool " << pool_name;
stats->Dequeue(false);
const UniqueIdPB& query_id = queue_node->admission_request.query_id;
+ const TQueryExecRequest* exec_req = nullptr;
+ Status status =
+ queue_node->admission_request.request.GetQueryExecRequest(&exec_req);
+ if (!status.ok()) {
+ LOG(WARNING) << "Failed to get query execution request for dequeued query "
+ << PrintId(query_id) << ": " << status.GetDetail();
+ // Fall to rejection handling.
+ queue_node->not_admitted_reason =
+ Substitute("Failed to get query execution request: $0", status.GetDetail());
+ is_rejected = true;
+ }
if (is_rejected) {
AdmissionOutcome outcome =
queue_node->admit_outcome->Set(AdmissionOutcome::REJECTED);
@@ -2667,8 +2759,7 @@ void AdmissionController::TryDequeue() {
DCHECK(!is_rejected);
DCHECK(queue_node->admitted_schedule != nullptr);
string user;
- Status status = GetEffectiveShortUser(
- queue_node->admission_request.request.query_ctx.session, &user);
+ status = GetEffectiveShortUser(exec_req->query_ctx.session, &user);
DCHECK_OK(status); // Should never happen because user name was checked at query
// entry.
AdmitQuery(queue_node, user, true /* was_queued */, is_trivial);
@@ -3097,15 +3188,15 @@ const pair<int64_t, int64_t> AdmissionController::GetAvailableMemAndSlots(
vector<const ExecutorGroup*> AdmissionController::GetExecutorGroupsForQuery(
const ClusterMembershipMgr::ExecutorGroups& all_groups,
- const AdmissionRequest& request) {
+ const TQueryExecRequest& exec_req) {
vector<const ExecutorGroup*> matching_groups;
- if (scheduler_->IsCoordinatorOnlyQuery(request.request)) {
+ if (scheduler_->IsCoordinatorOnlyQuery(exec_req)) {
// Coordinator only queries can run regardless of the presence of exec groups. This
// empty group works as a proxy to schedule coordinator only queries.
matching_groups.push_back(cluster_membership_mgr_->GetEmptyExecutorGroup());
return matching_groups;
}
- const string& pool_name = request.request.query_ctx.request_pool;
+ const string& pool_name = exec_req.query_ctx.request_pool;
string prefix(pool_name + POOL_GROUP_DELIMITER);
// We search for matching groups before the health check so that we don't fall back to
// the default group in case there are matching but unhealthy groups.
diff --git a/be/src/scheduling/admission-controller.h b/be/src/scheduling/admission-controller.h
index 6cf60c411..8be3b92be 100644
--- a/be/src/scheduling/admission-controller.h
+++ b/be/src/scheduling/admission-controller.h
@@ -34,6 +34,7 @@
#include "scheduling/schedule-state.h"
#include "statestore/statestore-subscriber.h"
#include "util/condition-variable.h"
+#include "util/debug-util.h"
#include "util/internal-queue.h"
#include "util/runtime-profile.h"
@@ -54,6 +55,82 @@ enum class AdmissionOutcome {
CANCELLED,
};
+class AdmissionExecRequest {
+ public:
+ virtual ~AdmissionExecRequest() = default;
+
+ /// Retrieves the underlying TQueryExecRequest.
+ /// For compressed requests, this may trigger decompression.
+ /// Returns a pointer to the request via the 'out_req' output parameter.
+ virtual Status GetQueryExecRequest(const TQueryExecRequest** out_req) const = 0;
+
+ /// Fast accessor for uncompressed requests only.
+ /// Avoids Status checks, DCHECKs if called on compressed requests.
+ virtual const TQueryExecRequest* request() const = 0;
+
+ /// For compressed only, clears the cached decompressed object to save memory.
+ virtual void ClearDecompressedCache() const = 0;
+};
+
+class AdmissionExecRequestUncompressed : public AdmissionExecRequest {
+ public:
+ explicit AdmissionExecRequestUncompressed(const TQueryExecRequest* req) : req_(req) {
+ DCHECK(req_ != nullptr);
+ }
+
+ Status GetQueryExecRequest(const TQueryExecRequest** out_req) const override {
+ *out_req = req_;
+ return Status::OK();
+ }
+
+ const TQueryExecRequest* request() const override {
+ DCHECK(req_ != nullptr);
+ return req_;
+ }
+
+ void ClearDecompressedCache() const override {
+ // we don't have a compressed data, so don't need to do anything.
+ }
+
+ private:
+ const TQueryExecRequest* req_;
+};
+
+class AdmissionController;
+class AdmissionExecRequestCompressed : public AdmissionExecRequest {
+ public:
+ explicit AdmissionExecRequestCompressed(
+ const TQueryExecRequestCompressed* req, AdmissionController* admission_controller)
+ : req_(req), admission_controller_(admission_controller) {
+ DCHECK(req_ != nullptr);
+ DCHECK(admission_controller_ != nullptr);
+ }
+
+ Status GetQueryExecRequest(const TQueryExecRequest** out_req) const override;
+
+ const TQueryExecRequest* request() const override {
+ DCHECK(false) << "AdmissionExecRequestCompressed does not support request(). "
+ << "Use GetQueryExecRequest() instead.";
+ return nullptr;
+ }
+
+ void ClearDecompressedCache() const override {
+ std::lock_guard<std::mutex> l(lock_);
+ if (decompressed_req_) {
+ LOG(INFO) << "Cleared the decompressed request for query "
+ << PrintId(decompressed_req_->query_ctx.query_id);
+ decompressed_req_.reset();
+ }
+ }
+
+ private:
+ const TQueryExecRequestCompressed* req_;
+ const AdmissionController* admission_controller_;
+ mutable std::mutex lock_;
+ mutable std::unique_ptr<TQueryExecRequest> decompressed_req_;
+ mutable bool first_decompress_ = true;
+};
+
/// The AdmissionController is used to throttle requests (e.g. queries, DML) based
/// on available cluster resources, which are configured in one or more resource pools. A
/// request will either be admitted for immediate execution, queued for later execution,
@@ -373,8 +450,7 @@ class AdmissionController {
struct AdmissionRequest {
const UniqueIdPB& query_id;
const UniqueIdPB& coord_id;
- const TQueryExecRequest& request;
- const TQueryOptions& query_options;
+ const AdmissionExecRequest& request;
RuntimeProfile* summary_profile;
std::unordered_set<NetworkAddressPB>& blacklisted_executor_addresses;
};
@@ -493,6 +569,9 @@ class AdmissionController {
admission_map_cleanup_cb_ = std::move(cb);
}
+ void UpdateAdmissionRequestCompressionStats(
+ int64_t compressed_size, int64_t uncompressed_size, float ratio) const;
+
private:
class PoolStats;
friend class PoolStats;
@@ -546,6 +625,12 @@ class AdmissionController {
/// executor groups).
IntCounter* total_dequeue_failed_coordinator_limited_ = nullptr;
+ /// Histograms for tracking the compressed/uncompressed sizes and compression ratios
+ /// of the admission requests of TQueryExecRequest.
+ mutable HistogramMetric* compressed_size_metric_;
+ mutable HistogramMetric* uncompressed_size_metric_;
+ mutable HistogramMetric* compression_ratio_metric_;
+
/// A typedef for a holder of per-user loads.
/// This matches the thrift-generated type of user_loads in TPoolStats.
/// There are a few helper functions for this type.
@@ -1328,7 +1413,7 @@ class AdmissionController {
/// resource pool associated with the query.
std::vector<const ExecutorGroup*> GetExecutorGroupsForQuery(
const ClusterMembershipMgr::ExecutorGroups& all_groups,
- const AdmissionRequest& request);
+ const TQueryExecRequest& exec_req);
/// Returns the current size of the cluster.
int64_t GetClusterSize(const ClusterMembershipMgr::Snapshot& membership_snapshot);
diff --git a/be/src/scheduling/remote-admission-control-client.cc b/be/src/scheduling/remote-admission-control-client.cc
index c090c89c9..f88d36ca0 100644
--- a/be/src/scheduling/remote-admission-control-client.cc
+++ b/be/src/scheduling/remote-admission-control-client.cc
@@ -17,6 +17,7 @@
#include "scheduling/remote-admission-control-client.h"
+#include "common/names.h"
#include "gen-cpp/admission_control_service.pb.h"
#include "gen-cpp/admission_control_service.proxy.h"
#include "kudu/rpc/rpc_controller.h"
@@ -31,15 +32,16 @@
#include "util/time.h"
#include "util/uid-util.h"
-#include "common/names.h"
-
-
DEFINE_int32(admission_status_retry_time_ms, 10,
"(Advanced) The number of milliseconds coordinators will wait before retrying the "
"GetQueryStatus rpc.");
DEFINE_int32(admission_max_retry_time_s, 60,
"(Advanced) The amount of time in seconds the coordinator will spend attempting to "
"retry admission if the admissiond is unreachable.");
+DEFINE_int64(admission_control_rpc_compress_threshold_bytes, 0,
+ "The minimum size in bytes of the serialized TQueryExecRequest required to trigger "
+ "compression in Admission Control RPCs. If set to 0, compression is disabled. "
+ "If set to a positive value, requests larger than this size will be compressed.");
using namespace strings;
using namespace kudu::rpc;
@@ -57,9 +59,12 @@ Status RemoteAdmissionControlClient::TryAdmitQuery(AdmissionControlServiceProxy*
AdmitQueryResponsePB resp;
RpcController rpc_controller;
- KrpcSerializer serializer;
+ KrpcSerializer serializer(FLAGS_admission_control_rpc_compress_threshold_bytes);
int sidecar_idx;
- RETURN_IF_ERROR(serializer.SerializeToSidecar(&request, &rpc_controller, &sidecar_idx));
+ bool is_compressed;
+ RETURN_IF_ERROR(serializer.SerializeToSidecar<TQueryExecRequestCompressed>(
+ &request, &rpc_controller, &sidecar_idx, &is_compressed));
+ req->set_is_compressed(is_compressed);
req->set_query_exec_request_sidecar_idx(sidecar_idx);
Status admit_status = Status::OK();
@@ -118,7 +123,7 @@ Status RemoteAdmissionControlClient::SubmitForAdmission(
int64_t admission_start = MonotonicMillis();
kudu::Status admit_rpc_status = kudu::Status::OK();
Status admit_status =
- TryAdmitQuery(proxy.get(), request.request, &req, &admit_rpc_status);
+ TryAdmitQuery(proxy.get(), *request.request.request(), &req, &admit_rpc_status);
int32_t num_retries = 0;
// Only retry AdmitQuery if the rpc layer reported a network error, indicating that the
// admissiond was unreachable.
@@ -142,7 +147,8 @@ Status RemoteAdmissionControlClient::SubmitForAdmission(
// Re-resolve the admissiond address on each retry to handle cases
// where the admissiond has restarted with a new IP.
RETURN_IF_ERROR(AdmissionControlService::GetProxy(&proxy));
- admit_status = TryAdmitQuery(proxy.get(), request.request, &req, &admit_rpc_status);
+ admit_status =
+ TryAdmitQuery(proxy.get(), *request.request.request(), &req, &admit_rpc_status);
}
KUDU_RETURN_IF_ERROR(admit_rpc_status, "AdmitQuery rpc failed");
diff --git a/be/src/service/client-request-state.cc b/be/src/service/client-request-state.cc
index 031a90348..d75ca6ee0 100644
--- a/be/src/service/client-request-state.cc
+++ b/be/src/service/client-request-state.cc
@@ -677,10 +677,10 @@ void ClientRequestState::FinishExecQueryOrDmlRequest() {
otel_span_manager_->StartChildSpanAdmissionControl();
}
- const TQueryExecRequest* query_exec_request;
TQueryExecRequest req;
+ req = exec_req.query_exec_request;
+ req.query_ctx.client_request.query_options = exec_req.query_options;
if (ExecEnv::GetInstance()->AdmissionServiceEnabled()) {
- req = exec_req.query_exec_request;
if (req.__isset.query_plan) {
// Use the swap() to ensure the string's memory is deallocated.
// Using clear() sets the size to 0 but may not release the capacity.
@@ -705,14 +705,13 @@ void ClientRequestState::FinishExecQueryOrDmlRequest() {
std::string().swap(client_req.redacted_stmt);
client_req.__isset.redacted_stmt = false;
}
- query_exec_request = &req;
- } else {
- query_exec_request = &exec_req.query_exec_request;
}
+ DCHECK(!admission_exec_request_);
+ admission_exec_request_ = std::make_unique<AdmissionExecRequestUncompressed>(&req);
Status admit_status = admission_control_client_->SubmitForAdmission(
- {query_id_pb, ExecEnv::GetInstance()->backend_id(), *query_exec_request,
- exec_req.query_options, summary_profile_, blacklisted_executor_addresses_},
+ {query_id_pb, ExecEnv::GetInstance()->backend_id(), *admission_exec_request_,
+ summary_profile_, blacklisted_executor_addresses_},
query_events_, &schedule_, &wait_start_time_ms_, &wait_end_time_ms_,
otel_span_manager_.get());
diff --git a/be/src/service/client-request-state.h b/be/src/service/client-request-state.h
index e327907e3..467e5aa55 100644
--- a/be/src/service/client-request-state.h
+++ b/be/src/service/client-request-state.h
@@ -39,6 +39,7 @@
namespace impala {
class AdmissionControlClient;
+class AdmissionExecRequest;
class Coordinator;
class Expr;
class Frontend;
@@ -757,6 +758,9 @@ class ClientRequestState {
/// is initialized once it's finalized. It's owned by the parent QueryDriver.
AtomicPtr<const TExecRequest> exec_request_{&unknown_exec_request_};
+ /// Construct only once to be used as a member of AdmissionRequest.
+ std::unique_ptr<AdmissionExecRequest> admission_exec_request_;
+
/// If true, effective_user() has access to the runtime profile and execution
/// summary.
AtomicBool user_has_profile_access_{true};
diff --git a/common/protobuf/admission_control_service.proto b/common/protobuf/admission_control_service.proto
index 4954ac572..201769ac0 100644
--- a/common/protobuf/admission_control_service.proto
+++ b/common/protobuf/admission_control_service.proto
@@ -219,6 +219,9 @@ message AdmitQueryRequestPB {
// List of backends this query should not be scheduled on.
repeated NetworkAddressPB blacklisted_executor_addresses = 4;
+
+ // Indicates whether the TQueryExecRequest sidecar is compressed.
+ optional bool is_compressed = 5 [default = false];
}
message AdmitQueryResponsePB {
diff --git a/common/thrift/Query.thrift b/common/thrift/Query.thrift
index 26750f5a1..ff46f940d 100644
--- a/common/thrift/Query.thrift
+++ b/common/thrift/Query.thrift
@@ -1079,6 +1079,14 @@ struct TFinalizeParams {
9: optional TIcebergDmlFinalizeParams iceberg_params;
}
+struct TQueryExecRequestCompressed {
+ // The size in bytes of the serialized TQueryExecRequest before compression.
+ 1: required i64 uncompressed_size
+
+ // Serialized TQueryExecRequest compressed data.
+ 2: required binary compressed_data
+}
+
// Result of call to ImpalaPlanService/JniFrontend.createExecRequest()
struct TQueryExecRequest {
// Exec info for all plans; the first one materializes the query result, and subsequent
diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json
index b5954b2e3..c358b699b 100644
--- a/common/thrift/metrics.json
+++ b/common/thrift/metrics.json
@@ -3958,6 +3958,36 @@
"kind": "COUNTER",
"key": "admission-controller.total-dequeue-failed-coordinator-limited"
},
+ {
+ "description": "Compressed TQueryExecRequest sizes processed by the Admission Controller.",
+ "contexts": [
+ "ADMISSIOND"
+ ],
+ "label": "Admission Control Query Execution Request Compressed Size",
+ "units": "BYTES",
+ "kind": "HISTOGRAM",
+ "key": "admission-controller.query-exec-request-compressed-bytes"
+ },
+ {
+ "description": "Uncompressed TQueryExecRequest sizes processed by the Admission Controller.",
+ "contexts": [
+ "ADMISSIOND"
+ ],
+ "label": "Admission Control Query Execution Request Uncompressed Size",
+ "units": "BYTES",
+ "kind": "HISTOGRAM",
+ "key": "admission-controller.query-exec-request-uncompressed-bytes"
+ },
+ {
+ "description": "Compression ratios (Uncompressed / Compressed) of TQueryExecRequest sizes.",
+ "contexts": [
+ "ADMISSIOND"
+ ],
+ "label": "Admission Control Query Execution Request Compression Ratio",
+ "units": "NONE",
+ "kind": "HISTOGRAM",
+ "key": "admission-controller.query-exec-request-compression-ratio"
+ },
{
"key": "admission-control-service.num-queries-high-water-mark",
"label": "HWM Num Queries in Admission Control Service",
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index 24c063a4a..4be8f7a76 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -2097,6 +2097,12 @@ class TestAdmissionControllerWithACService(TestAdmissionController):
"""Runs all of the tests from TestAdmissionController but with the second impalad in the
minicluster configured to perform all admission control."""
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestAdmissionControllerWithACService, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.add_dimension(
+ ImpalaTestDimension('admission_control_rpc_compress_threshold_bytes', 0, 1))
+
def get_ac_process(self):
return self.cluster.admissiond
(impala) 04/04: IMPALA-14692: Fix test_spilling_hash_join
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 00c233cc4fc25d23fc8a7e2f1efdf2d85c29f653
Author: Daniel Vanko <dv...@cloudera.com>
AuthorDate: Wed Jan 21 15:14:52 2026 +0100
IMPALA-14692: Fix test_spilling_hash_join
IMPALA-14680 mistakenly removed greedy regex patterns from
query-impala-13138.test, but this test checks for the query results not
for the query profile, which was modified in IMPALA-14680.
Testing:
* test_spilling_hash_join passed in exhaustive mode
Change-Id: I709f81217f44c9377e4a1e8419787591ba7b7451
Reviewed-on: http://gerrit.cloudera.org:8080/23898
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../functional-query/queries/QueryTest/query-impala-13138.test | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/testdata/workloads/functional-query/queries/QueryTest/query-impala-13138.test b/testdata/workloads/functional-query/queries/QueryTest/query-impala-13138.test
index b80f4dff9..d3f655ba7 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/query-impala-13138.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/query-impala-13138.test
@@ -23,7 +23,7 @@ FROM letter_marketing_response_events re
AND re.send_account_sk not in (43)
GROUP BY ecc.letter_mission_name, re.`date`, c.war_group;
---- RESULTS: VERIFY_IS_SUBSET
-row_regex:,'2024-06-07',
+row_regex:.*,'2024-06-07',.*
---- TYPES
STRING,STRING,STRING
====
(impala) 01/04: IMPALA-14525: Calcite planner: Add support for RexSimplify
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 2360a06e4ac0ab27d6f3dcec1c366a3ba4904089
Author: Steve Carlin <sc...@cloudera.com>
AuthorDate: Tue Nov 4 05:21:22 2025 -0800
IMPALA-14525: Calcite planner: Add support for RexSimplify
RexSimplify is a class in Calcite that simplifies expressions
into something more optimal. It was disabled up until this point
because it converts IN clauses into a Calcite internal SEARCH
object which isn't directly supported by Impala.
This commit brings back the RexSimplify class. The SEARCH
operator is now converted into an IN operator when RexNode
objects are changed into Expr objects.
Some notes about the changes that had to be made:
- some small refactoring needed to be done in the Impala Expr
objects.
- RexSimplify is very stringent about operators that are nullable,
as there is an assert when certain operators are checked. There is
logic in the CoerceOperandShuttle that ensures the nullability is
now set correctly.
- Some duplicated logic at line 148 in CoerceOperandShuttle was removed,
(existing logic in getReturnType)
- The AnalyzedInPredicate subclass was created to avoid analysis done in
InPredicate.
- Removed ImpalaRexBuilder logic which avoided creation of the SEARCH op.
- Created ImpalaRexSimplify which extends RexSimplify. RexSimplify causes
regressions with NaN on comparisons with Double. For instance,
"where not(my_col > 30)" changes to "where my_col <= 30": The first
expression returns true when my_col is NaN and the second expression
returns false. So ImpalaRexSimplify looks for the existence of any
binary comparison operator with Double in it and avoids the simplification.
- Added ImpalaRexUtil which copies the RexUtil.expandSearch() method that
converts the SEARCH operator into non-search operators. The version here
handles the conversion to the custom Impala IN operator.
- Created an ImpalaCoreRules class. Even though RexSimplify is supported,
it is important it is run through ImpalaRexSimplify. The RexSimplify
is disabled for the SqlToRelNode converter and for all rules given by
Calcite. ImpalaCoreRules also has the benefit of having one place where
one can find all the rules used by Impala.
- Created simplify rules for the filter condition, and the projects in
the project object.
- Changed the FilterSelectivityEstimator to get the selectivity for the
SEARCH operator.
- Added a couple of rules in the optimizer for a bug that was being exposed
when enabling the SEARCH operator. The PROJECT_JOIN_TRANSPOSE was removed
because it did not serve any purpose, as we transpose JOIN_PROJECT in the
join phase. Some other rules were added to help with pushdown predicates
like JOIN_DERIVE_IS_NOT_NULL_FILTER and JOIN_PUSH_EXPRESSIONS. And the
Simplifier rules have also been added.
- Some of the new rules caused many changes in the estimations of cardinality
and memory. The one noticeable change was using IsNullPredicate for the
IS_NULL and IS_NOT_NULL operators. Previously, these functions were using
FunctionCallExpr, and the cardinality estimation was way off.
- Fixed a small bug in RexLiteralConverter where a string literal was treated
as a VARCHAR. A string literal should always be treated as a STRING.
Change-Id: I44792688f361bf15affa565e5de5709f64dcf18c
Reviewed-on: http://gerrit.cloudera.org:8080/23679
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Michael Smith <mi...@cloudera.com>
Reviewed-by: Aman Sinha <am...@cloudera.com>
---
.../apache/impala/analysis/FunctionCallExpr.java | 2 +-
.../org/apache/impala/analysis/InPredicate.java | 28 +-
.../calcite/coercenodes/CoerceOperandShuttle.java | 60 +-
.../calcite/functions/AnalyzedInPredicate.java | 80 +
.../impala/calcite/functions/RexCallConverter.java | 20 +
.../calcite/functions/RexLiteralConverter.java | 8 +-
.../calcite/operators/ImpalaCastFunction.java | 6 +-
.../impala/calcite/operators/ImpalaInOperator.java | 37 +
.../impala/calcite/operators/ImpalaRexBuilder.java | 56 -
.../calcite/operators/ImpalaRexSimplify.java | 81 +
.../impala/calcite/operators/ImpalaRexUtil.java | 228 +
.../impala/calcite/rel/node/ImpalaJoinRel.java | 5 +-
.../impala/calcite/rel/util/CreateExprVisitor.java | 13 +-
.../calcite/rel/util/ExprConjunctsConverter.java | 14 +-
.../impala/calcite/rules/ImpalaCoreRules.java | 209 +
.../calcite/rules/ImpalaFilterSimplifyRule.java | 62 +
.../calcite/rules/ImpalaProjectSimplifyRule.java | 78 +
.../calcite/schema/FilterSelectivityEstimator.java | 30 +-
.../impala/calcite/service/CalciteOptimizer.java | 120 +-
.../calcite/service/CalciteRelNodeConverter.java | 32 +-
.../impala/calcite/type/ImpalaTypeConverter.java | 16 +-
.../calcite/analytic-rank-pushdown-calcite.test | 124 +-
.../calcite/limit-pushdown-analytic-calcite.test | 2 +-
.../PlannerTest/calcite_tpcds/tpcds-q01.test | 403 +-
.../PlannerTest/calcite_tpcds/tpcds-q02.test | 246 +-
.../PlannerTest/calcite_tpcds/tpcds-q03.test | 141 +-
.../PlannerTest/calcite_tpcds/tpcds-q04.test | 2087 +++---
.../PlannerTest/calcite_tpcds/tpcds-q05.test | 480 +-
.../PlannerTest/calcite_tpcds/tpcds-q06.test | 767 +--
.../PlannerTest/calcite_tpcds/tpcds-q07.test | 178 +-
.../PlannerTest/calcite_tpcds/tpcds-q08.test | 762 ++-
.../PlannerTest/calcite_tpcds/tpcds-q10a.test | 1165 ++--
.../PlannerTest/calcite_tpcds/tpcds-q11.test | 788 +--
.../PlannerTest/calcite_tpcds/tpcds-q12.test | 128 +-
.../PlannerTest/calcite_tpcds/tpcds-q13.test | 569 +-
.../PlannerTest/calcite_tpcds/tpcds-q14a.test | 6264 +++++++++---------
.../PlannerTest/calcite_tpcds/tpcds-q14b.test | 3434 +++++-----
.../PlannerTest/calcite_tpcds/tpcds-q15.test | 145 +-
.../PlannerTest/calcite_tpcds/tpcds-q16.test | 1608 ++---
.../PlannerTest/calcite_tpcds/tpcds-q17.test | 723 ++-
.../PlannerTest/calcite_tpcds/tpcds-q18.test | 347 +-
.../PlannerTest/calcite_tpcds/tpcds-q19.test | 213 +-
.../PlannerTest/calcite_tpcds/tpcds-q20.test | 146 +-
.../PlannerTest/calcite_tpcds/tpcds-q21.test | 297 +-
.../PlannerTest/calcite_tpcds/tpcds-q22.test | 59 +-
.../PlannerTest/calcite_tpcds/tpcds-q23a.test | 3586 +++++------
.../PlannerTest/calcite_tpcds/tpcds-q23b.test | 3759 +++++------
.../PlannerTest/calcite_tpcds/tpcds-q24a.test | 744 +--
.../PlannerTest/calcite_tpcds/tpcds-q24b.test | 744 +--
.../PlannerTest/calcite_tpcds/tpcds-q25.test | 719 ++-
.../PlannerTest/calcite_tpcds/tpcds-q26.test | 178 +-
.../PlannerTest/calcite_tpcds/tpcds-q27.test | 232 +-
.../PlannerTest/calcite_tpcds/tpcds-q29.test | 571 +-
.../PlannerTest/calcite_tpcds/tpcds-q30.test | 611 +-
.../PlannerTest/calcite_tpcds/tpcds-q31.test | 2602 ++++----
.../PlannerTest/calcite_tpcds/tpcds-q32.test | 661 +-
.../PlannerTest/calcite_tpcds/tpcds-q33.test | 1397 ++--
.../PlannerTest/calcite_tpcds/tpcds-q34.test | 505 +-
.../PlannerTest/calcite_tpcds/tpcds-q35a.test | 1339 ++--
.../PlannerTest/calcite_tpcds/tpcds-q36.test | 156 +-
.../PlannerTest/calcite_tpcds/tpcds-q37.test | 227 +-
.../PlannerTest/calcite_tpcds/tpcds-q38.test | 429 +-
.../PlannerTest/calcite_tpcds/tpcds-q39a.test | 128 +-
.../PlannerTest/calcite_tpcds/tpcds-q39b.test | 128 +-
.../PlannerTest/calcite_tpcds/tpcds-q40.test | 331 +-
.../PlannerTest/calcite_tpcds/tpcds-q41.test | 91 +-
.../PlannerTest/calcite_tpcds/tpcds-q42.test | 119 +-
.../calcite_tpcds/tpcds-q43-verbose.test | 143 +-
.../PlannerTest/calcite_tpcds/tpcds-q43.test | 283 +-
.../PlannerTest/calcite_tpcds/tpcds-q44.test | 202 +-
.../PlannerTest/calcite_tpcds/tpcds-q45.test | 1031 +--
.../PlannerTest/calcite_tpcds/tpcds-q46.test | 408 +-
.../PlannerTest/calcite_tpcds/tpcds-q47.test | 981 +--
.../PlannerTest/calcite_tpcds/tpcds-q48.test | 444 +-
.../PlannerTest/calcite_tpcds/tpcds-q49.test | 531 +-
.../PlannerTest/calcite_tpcds/tpcds-q50.test | 218 +-
.../PlannerTest/calcite_tpcds/tpcds-q51.test | 416 +-
.../PlannerTest/calcite_tpcds/tpcds-q52.test | 131 +-
.../PlannerTest/calcite_tpcds/tpcds-q53.test | 172 +-
.../PlannerTest/calcite_tpcds/tpcds-q54.test | 391 +-
.../PlannerTest/calcite_tpcds/tpcds-q55.test | 131 +-
.../PlannerTest/calcite_tpcds/tpcds-q56.test | 1258 ++--
.../PlannerTest/calcite_tpcds/tpcds-q57.test | 1063 ++--
.../PlannerTest/calcite_tpcds/tpcds-q58.test | 2615 ++++----
.../PlannerTest/calcite_tpcds/tpcds-q59.test | 260 +-
.../PlannerTest/calcite_tpcds/tpcds-q60.test | 1283 ++--
.../PlannerTest/calcite_tpcds/tpcds-q61.test | 435 +-
.../PlannerTest/calcite_tpcds/tpcds-q62.test | 98 +-
.../PlannerTest/calcite_tpcds/tpcds-q63.test | 180 +-
.../PlannerTest/calcite_tpcds/tpcds-q64.test | 6636 ++++++++++----------
.../PlannerTest/calcite_tpcds/tpcds-q65.test | 255 +-
.../PlannerTest/calcite_tpcds/tpcds-q66.test | 668 +-
.../PlannerTest/calcite_tpcds/tpcds-q67.test | 179 +-
.../PlannerTest/calcite_tpcds/tpcds-q68.test | 329 +-
.../PlannerTest/calcite_tpcds/tpcds-q69.test | 1721 +++--
.../PlannerTest/calcite_tpcds/tpcds-q70.test | 282 +-
.../PlannerTest/calcite_tpcds/tpcds-q71.test | 356 +-
.../PlannerTest/calcite_tpcds/tpcds-q72.test | 412 +-
.../PlannerTest/calcite_tpcds/tpcds-q73.test | 292 +-
.../PlannerTest/calcite_tpcds/tpcds-q74.test | 2128 ++++---
.../PlannerTest/calcite_tpcds/tpcds-q75.test | 1446 +++--
.../PlannerTest/calcite_tpcds/tpcds-q76.test | 480 +-
.../PlannerTest/calcite_tpcds/tpcds-q77.test | 744 ++-
.../PlannerTest/calcite_tpcds/tpcds-q78.test | 425 +-
.../PlannerTest/calcite_tpcds/tpcds-q79.test | 244 +-
.../PlannerTest/calcite_tpcds/tpcds-q80.test | 517 +-
.../PlannerTest/calcite_tpcds/tpcds-q81.test | 631 +-
.../PlannerTest/calcite_tpcds/tpcds-q82.test | 181 +-
.../PlannerTest/calcite_tpcds/tpcds-q83.test | 2393 +++----
.../PlannerTest/calcite_tpcds/tpcds-q84.test | 212 +-
.../PlannerTest/calcite_tpcds/tpcds-q85.test | 775 +--
.../PlannerTest/calcite_tpcds/tpcds-q86.test | 89 +-
.../PlannerTest/calcite_tpcds/tpcds-q87.test | 429 +-
.../PlannerTest/calcite_tpcds/tpcds-q88.test | 2770 ++++----
.../PlannerTest/calcite_tpcds/tpcds-q89.test | 254 +-
.../PlannerTest/calcite_tpcds/tpcds-q90.test | 302 +-
.../PlannerTest/calcite_tpcds/tpcds-q91.test | 286 +-
.../PlannerTest/calcite_tpcds/tpcds-q92.test | 649 +-
.../PlannerTest/calcite_tpcds/tpcds-q93.test | 412 +-
.../PlannerTest/calcite_tpcds/tpcds-q94.test | 1779 +++---
.../PlannerTest/calcite_tpcds/tpcds-q95.test | 1553 ++---
.../PlannerTest/calcite_tpcds/tpcds-q96.test | 348 +-
.../PlannerTest/calcite_tpcds/tpcds-q97.test | 248 +-
.../PlannerTest/calcite_tpcds/tpcds-q98.test | 146 +-
.../PlannerTest/calcite_tpcds/tpcds-q99.test | 98 +-
125 files changed, 44580 insertions(+), 40281 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/FunctionCallExpr.java b/fe/src/main/java/org/apache/impala/analysis/FunctionCallExpr.java
index 2933d17b5..a181e8617 100644
--- a/fe/src/main/java/org/apache/impala/analysis/FunctionCallExpr.java
+++ b/fe/src/main/java/org/apache/impala/analysis/FunctionCallExpr.java
@@ -455,7 +455,7 @@ public class FunctionCallExpr extends Expr {
}
@Override
- protected boolean isConstantImpl() {
+ public boolean isConstantImpl() {
// TODO: we can't correctly determine const-ness before analyzing 'fn_'. We should
// rework logic so that we do not call this function on unanalyzed exprs.
// Aggregate functions are never constant.
diff --git a/fe/src/main/java/org/apache/impala/analysis/InPredicate.java b/fe/src/main/java/org/apache/impala/analysis/InPredicate.java
index ca175f84d..10ee64188 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InPredicate.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InPredicate.java
@@ -20,6 +20,7 @@ package org.apache.impala.analysis;
import java.util.List;
import org.apache.impala.catalog.Db;
+import org.apache.impala.catalog.Function;
import org.apache.impala.catalog.Function.CompareMode;
import org.apache.impala.catalog.PrimitiveType;
import org.apache.impala.catalog.ScalarFunction;
@@ -149,22 +150,11 @@ public class InPredicate extends Predicate {
break;
}
}
- boolean useSetLookup = allConstant;
- // Threshold based on InPredicateBenchmark results
- int setLookupThreshold = children_.get(0).getType().isStringType() ? 2 : 6;
- if (children_.size() - 1 < setLookupThreshold) useSetLookup = false;
// Only lookup fn_ if all subqueries have been rewritten. If the second child is a
// subquery, it will have type ArrayType, which cannot be resolved to a builtin
// function and will fail analysis.
- Type[] argTypes = {getChild(0).type_, getChild(1).type_};
- if (useSetLookup) {
- fn_ = getBuiltinFunction(analyzer, isNotIn_ ? NOT_IN_SET_LOOKUP : IN_SET_LOOKUP,
- argTypes, CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
- } else {
- fn_ = getBuiltinFunction(analyzer, isNotIn_ ? NOT_IN_ITERATE : IN_ITERATE,
- argTypes, CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
- }
+ fn_ = getFunction(analyzer, allConstant);
Preconditions.checkNotNull(fn_);
Preconditions.checkState(fn_.getReturnType().isBoolean());
castForFunctionCall(false, analyzer.getRegularCompatibilityLevel());
@@ -172,6 +162,20 @@ public class InPredicate extends Predicate {
computeSelectivity();
}
+ protected Function getFunction(Analyzer analyzer, boolean isAllConstant)
+ throws AnalysisException {
+ // Threshold based on InPredicateBenchmark results
+ int setLookupThreshold = children_.get(0).getType().isStringType() ? 2 : 6;
+ boolean useSetLookup = isAllConstant && (children_.size() - 1 >= setLookupThreshold);
+
+ Type[] argTypes = {getChild(0).type_, getChild(1).type_};
+ return useSetLookup
+ ? getBuiltinFunction(analyzer, isNotIn_ ? NOT_IN_SET_LOOKUP : IN_SET_LOOKUP,
+ argTypes, CompareMode.IS_NONSTRICT_SUPERTYPE_OF)
+ : getBuiltinFunction(analyzer, isNotIn_ ? NOT_IN_ITERATE : IN_ITERATE,
+ argTypes, CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+ }
+
protected void computeSelectivity() {
if (hasValidSelectivityHint()) {
return;
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/coercenodes/CoerceOperandShuttle.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/coercenodes/CoerceOperandShuttle.java
index a241eaf95..d2a1a57cb 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/coercenodes/CoerceOperandShuttle.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/coercenodes/CoerceOperandShuttle.java
@@ -21,6 +21,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
+import org.apache.calcite.plan.Strong;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
@@ -96,9 +97,9 @@ public class CoerceOperandShuttle extends RexShuttle {
@Override
public RexNode visitCall(RexCall call) {
- // Eliminate the "Sarg" function which is unknown to Impala.
- // TODO: this is kinda hacky. It would be better if Impala can handle this
- // directly, so this needs investigation.
+ // Eliminate the SEARCH operator while coercing. This is an internal Calcite
+ // operator used in simplifications. It will be brought back after the coercing
+ // is complete.
if (call.getOperator().getKind().equals(SqlKind.SEARCH)) {
return visitCall((RexCall) RexUtil.expandSearch(rexBuilder, null, call));
}
@@ -135,7 +136,8 @@ public class CoerceOperandShuttle extends RexShuttle {
call);
}
- RelDataType retType = getReturnType(castedOperandsCall, fn.getReturnType());
+ RelDataType retType =
+ getReturnType(rexBuilder, castedOperandsCall, fn.getReturnType());
// This code does not handle changes in the return type when the Calcite
// function is not a decimal but the function resolves to a function that
@@ -145,12 +147,6 @@ public class CoerceOperandShuttle extends RexShuttle {
Preconditions.checkState(!SqlTypeUtil.isDecimal(retType) ||
SqlTypeUtil.isDecimal(castedOperandsCall.getType()));
- // So if the original return type is Decimal and the function resolves to
- // decimal, the precision and scale are saved from the original function.
- if (SqlTypeUtil.isDecimal(retType)) {
- retType = castedOperandsCall.getType();
- }
-
List<RexNode> newOperands = getCastedArgTypes(fn, castedOperandsCall.getOperands(),
retType, factory, rexBuilder);
@@ -174,7 +170,7 @@ public class CoerceOperandShuttle extends RexShuttle {
over);
}
- RelDataType retType = getReturnType(castedOver, fn.getReturnType());
+ RelDataType retType = getReturnType(rexBuilder, castedOver, fn.getReturnType());
List<RexNode> newOperands =
getCastedArgTypes(fn, castedOver.getOperands(), retType, factory, rexBuilder);
@@ -219,7 +215,8 @@ public class CoerceOperandShuttle extends RexShuttle {
}
- private RelDataType getReturnType(RexCall rexCall, Type impalaReturnType) {
+ private RelDataType getReturnType(RexBuilder rexBuilder, RexCall rexCall,
+ Type impalaReturnType) {
// Case is a special case. Currently, there is a quirk in the Impala function
// resolver where it always returns the BOOLEAN signature. So the return type
// is evaluated here by finding the compatible type amongst the "then" clauses.
@@ -229,7 +226,9 @@ public class CoerceOperandShuttle extends RexShuttle {
return ImpalaTypeConverter.getCompatibleTypeForCase(argTypes, factory);
}
- RelDataType retType = ImpalaTypeConverter.getRelDataType(impalaReturnType);
+ boolean isNullable = isNullable(rexCall);
+ RelDataType retType =
+ ImpalaTypeConverter.getRelDataType(impalaReturnType, isNullable);
// This code does not handle changes in the return type when the Calcite
// function is not a decimal but the function resolves to a function that
@@ -242,7 +241,8 @@ public class CoerceOperandShuttle extends RexShuttle {
// So if the original return type is Decimal and the function resolves to
// decimal, the precision and scale are saved from the original function.
if (SqlTypeUtil.isDecimal(retType)) {
- retType = rexCall.getType();
+ retType = rexBuilder.getTypeFactory().createTypeWithNullability(rexCall.getType(),
+ isNullable);
}
return retType;
@@ -343,7 +343,8 @@ public class CoerceOperandShuttle extends RexShuttle {
Type toImpalaType = fn.getArgs()[indexToUse];
RelDataType toType = useReturnTypeForCastingArg(fn, argTypes.get(indexToUse))
? retType
- : getCastedToType(argTypes.get(i), toImpalaType, factory);
+ : getCastedToType(argTypes.get(i), toImpalaType, factory,
+ isNullable(operands.get(i)));
RexNode operand = castOperand(operands.get(i), toType,
factory, rexBuilder);
@@ -357,6 +358,28 @@ public class CoerceOperandShuttle extends RexShuttle {
return castedOperand ? newOperands : operands;
}
+ /**
+ * Returns true if RexNode is nullable. Literals are only nullable if they contain
+ * the null value. Operators need to match what Calcite expects for RexSimplify to
+ * work properly. Any operators not found in Calcite are assumed to be nullable.
+ */
+ private static boolean isNullable(RexNode rexNode) {
+ if (rexNode instanceof RexLiteral) {
+ return ((RexLiteral) rexNode).isNull();
+ }
+ switch (Strong.policy(rexNode)) {
+ case NOT_NULL:
+ return false;
+ case ANY:
+ List<RexNode> operands = ((RexCall) rexNode).getOperands();
+ return ((RexCall) rexNode).getOperands().stream()
+ .map(RexNode::getType)
+ .anyMatch(RelDataType::isNullable);
+ default:
+ return true;
+ }
+ }
+
private static boolean useReturnTypeForCastingArg(Function fn, RelDataType argType) {
// case functions use the precalculated return type from the function resolver.
if (isCaseFunction(fn)) {
@@ -374,7 +397,7 @@ public class CoerceOperandShuttle extends RexShuttle {
}
private static RelDataType getCastedToType(RelDataType fromType,
- Type toImpalaType, RelDataTypeFactory factory) {
+ Type toImpalaType, RelDataTypeFactory factory, boolean isNullable) {
// Special case: If the "to" type is a generic CHAR (where len = -1),
// there is no casting needed if the "from" type is also a CHAR.
@@ -392,7 +415,7 @@ public class CoerceOperandShuttle extends RexShuttle {
}
if (!toImpalaType.isDecimal() || SqlTypeUtil.isNull(fromType)) {
- return ImpalaTypeConverter.getRelDataType(toImpalaType);
+ return ImpalaTypeConverter.getRelDataType(toImpalaType, isNullable);
}
// Integer based type needs special conversion to Decimal types based on the
@@ -400,8 +423,9 @@ public class CoerceOperandShuttle extends RexShuttle {
// the type if the from type is also DECIMAL.
ScalarType impalaType = (ScalarType) ImpalaTypeConverter.createImpalaType(fromType);
ScalarType decimalType = impalaType.getMinResolutionDecimal();
- return factory.createSqlType(SqlTypeName.DECIMAL,
+ RelDataType relDataType = factory.createSqlType(SqlTypeName.DECIMAL,
decimalType.decimalPrecision(), decimalType.decimalScale());
+ return factory.createTypeWithNullability(relDataType, isNullable);
}
/**
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/AnalyzedInPredicate.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/AnalyzedInPredicate.java
new file mode 100644
index 000000000..d81251d77
--- /dev/null
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/AnalyzedInPredicate.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.impala.calcite.functions;
+
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.impala.analysis.Analyzer;
+import org.apache.impala.analysis.Expr;
+import org.apache.impala.analysis.InPredicate;
+import org.apache.impala.catalog.Function;
+import org.apache.impala.catalog.Type;
+import org.apache.impala.common.AnalysisException;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * "In" Expression that is always in analyzed state
+ */
+public class AnalyzedInPredicate extends InPredicate {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(AnalyzedInPredicate.class.getName());
+
+ // Need to save the function because it is known at constructor time. The
+ // resetAnalyzeState() method can be called at various points which could
+ // set the fn_ member to null. So we save the function in the savedFunction_
+ // variable so it can be properly set in analyzeImpl()
+ private final Function savedFunction_;
+
+ public AnalyzedInPredicate(RexCall call, List<Expr> params, Analyzer analyzer)
+ throws AnalysisException {
+ super(params.get(0), params.subList(1, params.size()),
+ call.getKind().equals(SqlKind.NOT_IN));
+ this.savedFunction_ = getFunction(analyzer, true);
+
+ this.type_ = Type.BOOLEAN;
+ }
+
+ public AnalyzedInPredicate(AnalyzedInPredicate other) {
+ super(other);
+ this.savedFunction_ = other.savedFunction_;
+ }
+
+ @Override
+ public Expr clone() {
+ return new AnalyzedInPredicate(this);
+ }
+
+ @Override
+ protected void analyzeImpl(Analyzer analyzer) throws AnalysisException {
+ this.computeSelectivity();
+ this.fn_ = savedFunction_;
+ this.type_ = Type.BOOLEAN;
+ }
+
+ @Override
+ protected float computeEvalCost() {
+ // TODO: IMPALA-13098: need to implement
+ return 1;
+ }
+}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java
index 5cf50002e..da5513fa0 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java
@@ -35,7 +35,9 @@ import org.apache.impala.analysis.CaseWhenClause;
import org.apache.impala.analysis.CompoundPredicate;
import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.FunctionCallExpr;
+import org.apache.impala.analysis.IsNullPredicate;
import org.apache.impala.analysis.TimestampArithmeticExpr;
+import org.apache.impala.calcite.operators.ImpalaInOperator;
import org.apache.impala.calcite.type.ImpalaTypeConverter;
import org.apache.impala.catalog.Function;
import org.apache.impala.catalog.Type;
@@ -79,6 +81,16 @@ public class RexCallConverter {
return createCompoundExpr(rexCall, params);
case CAST:
return createCastExpr(rexCall, params, analyzer);
+ case NOT_IN:
+ return createInExpr(rexCall, params, analyzer);
+ case IS_NULL:
+ return new IsNullPredicate(params.get(0), false);
+ case IS_NOT_NULL:
+ return new IsNullPredicate(params.get(0), true);
+ case OTHER:
+ if (rexCall.getOperator() instanceof ImpalaInOperator) {
+ return createInExpr(rexCall, params, analyzer);
+ }
}
if (rexCall.getOperator().getName().toLowerCase().equals("explicit_cast")) {
@@ -164,6 +176,14 @@ public class RexCallConverter {
return null;
}
+ /**
+ * Create In Expr
+ */
+ private static Expr createInExpr(RexCall call, List<Expr> params, Analyzer analyzer
+ ) throws ImpalaException {
+ return new AnalyzedInPredicate(call, params, analyzer);
+ }
+
private static Expr createCastExpr(RexCall call, List<Expr> params, Analyzer analyzer)
throws ImpalaException {
Type impalaRetType = ImpalaTypeConverter.createImpalaType(call.getType());
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexLiteralConverter.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexLiteralConverter.java
index 1b768c1ca..8a0cc141e 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexLiteralConverter.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexLiteralConverter.java
@@ -95,12 +95,8 @@ public class RexLiteralConverter {
return numericExpr;
case CHAR:
case VARCHAR:
- ScalarType charType = rexLiteral.getType().getSqlTypeName() == SqlTypeName.VARCHAR
- ? Type.STRING
- : ScalarType.createCharType(rexLiteral.getValueAs(String.class).length());
- Expr charExpr = new StringLiteral(rexLiteral.getValueAs(String.class),
- charType, false);
- return charExpr;
+ // Always treat all string literals as type STRING
+ return new StringLiteral(rexLiteral.getValueAs(String.class), Type.STRING, false);
case DATE:
DateString dateStringClass = rexLiteral.getValueAs(DateString.class);
String dateString = (dateStringClass == null) ? null : dateStringClass.toString();
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaCastFunction.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaCastFunction.java
index e107bb88d..5352224a3 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaCastFunction.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaCastFunction.java
@@ -61,9 +61,9 @@ public class ImpalaCastFunction extends ImpalaOperator {
List<RelDataType> castFromList = operandTypes.subList(0, 1);
- return CommonOperatorFunctions.isNullable(castFromList)
- ? factory.createTypeWithNullability(operandTypes.get(1), true)
- : operandTypes.get(1);
+ // Cast function always has the potential to return null, so set the
+ // nullability to true to be consistent for all casts.
+ return factory.createTypeWithNullability(operandTypes.get(1), true);
}
private String getCastToName(RelDataType type) {
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaInOperator.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaInOperator.java
new file mode 100644
index 000000000..6a630dfe2
--- /dev/null
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaInOperator.java
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.calcite.operators;
+
+/**
+ * Special "In" operator for Impala.
+ *
+ * Calcite treats all IN operators as subqueries. Impala needs its own
+ * distinct operator to deal with the IN function;
+ */
+public class ImpalaInOperator extends ImpalaOperator {
+
+ public static final ImpalaInOperator OP = new ImpalaInOperator();
+
+ private ImpalaInOperator() {
+ // Using "in_set_lookup" for the function name. Impala needs a valid
+ // function here for the coerce nodes module to make sure all datatypes
+ // are correct. At physical conversion time, the actual function used
+ // may be changed to in_iterate based on certain conditions.
+ super("in_set_lookup");
+ }
+}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaRexBuilder.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaRexBuilder.java
deleted file mode 100644
index 2f374d136..000000000
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaRexBuilder.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.impala.calcite.operators;
-
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.SqlOperator;
-
-import java.util.List;
-
-/**
- * Factory for row expressions.
- *
- * See the base class in Calcite for the details about RexBuilder. This extension
- * allows Impala to override functions that need extra work to make RexBuilder
- * Impala compatible.
- */
-public class ImpalaRexBuilder extends RexBuilder {
-
- public ImpalaRexBuilder(RelDataTypeFactory typeFactory) {
- super(typeFactory);
- }
-
- /**
- * Only override the makeCall. The SEARCH operator is not supported at this time,
- * so we "expand" it to something Impala can understand.
- */
- @Override
- public RexNode makeCall(
- SqlOperator op,
- List<? extends RexNode> exprs) {
- RexNode retNode = super.makeCall(op, exprs);
- return op.getKind() == SqlKind.SEARCH
- ? RexUtil.expandSearch(this, null, retNode)
- : retNode;
- }
-}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaRexSimplify.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaRexSimplify.java
new file mode 100644
index 000000000..8260a93de
--- /dev/null
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaRexSimplify.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.impala.calcite.operators;
+
+import org.apache.calcite.plan.RelOptPredicateList;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexExecutor;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexSimplify;
+import org.apache.calcite.rex.RexUnknownAs;
+import org.apache.calcite.sql.SqlBinaryOperator;
+import org.apache.calcite.sql.type.SqlTypeUtil;
+
+
+/**
+ * ImpalaRexSimplify extends the RexSimplify class in order to 'simplify' expressions.
+ * The extension of the "simplify" method catches the double and float types used in
+ * a binary predicate and avoids the simplification. This is needed because Calcite
+ * will simplify various cases of NaN the wrong way. For example, NOT(my_col >= 3)
+ * should include NaN in its calculation. But this gets simplified to my_col < 3
+ * which does not include NaN in its calculation.
+ */
+public class ImpalaRexSimplify extends RexSimplify {
+ private final RexExecutor rexExecutor_;
+
+ public ImpalaRexSimplify(RexBuilder rexBuilder, RexExecutor rexExecutor) {
+ super(rexBuilder, RelOptPredicateList.EMPTY, rexExecutor);
+ rexExecutor_ = rexExecutor;
+ }
+
+ public RexExecutor getRexExecutor() {
+ return rexExecutor_;
+ }
+
+ @Override
+ public RexNode simplify(RexNode rexNode) {
+ return hasApproximateTypeIssues(rexNode)
+ ? rexNode
+ : super.simplifyUnknownAs(rexNode, RexUnknownAs.FALSE);
+ }
+
+ private boolean hasApproximateTypeIssues(RexNode rexNode) {
+ RexHasApproximateIssuesShuttle shuttle = new RexHasApproximateIssuesShuttle();
+ shuttle.apply(rexNode);
+ return shuttle.hasApproximateIssues_;
+ }
+
+ private static class RexHasApproximateIssuesShuttle extends RexShuttle {
+ public boolean hasApproximateIssues_ = false;
+
+ // Sets hasApproximateIssues to true if there is a binary predicate with a double
+ // or float type in it.
+ @Override
+ public RexNode visitCall(RexCall rexCall) {
+ super.visitCall(rexCall);
+ if (rexCall.getOperator() instanceof SqlBinaryOperator) {
+ if (SqlTypeUtil.isApproximateNumeric(rexCall.getOperands().get(0).getType()) ||
+ SqlTypeUtil.isApproximateNumeric(rexCall.getOperands().get(1).getType())) {
+ hasApproximateIssues_ = true;
+ }
+ }
+ return rexCall;
+ }
+ }
+}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaRexUtil.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaRexUtil.java
new file mode 100644
index 000000000..4cf020ab7
--- /dev/null
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaRexUtil.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.impala.calcite.operators;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexLocalRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexProgram;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexUnknownAs;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.util.RangeSets;
+import org.apache.calcite.util.Sarg;
+
+import java.util.ArrayList;
+import java.util.List;
+import static java.util.Objects.requireNonNull;
+
+/**
+ * ImpalaRexUtil: class for RexUtil type functions.
+ *
+ * This class contains the copied "expandSearch" method from Calcite's RexUtil method
+ * and all the code that is called from it. As of Calcite 1.40, there is no native
+ * IN operator (CALCITE-7232). Instead, Calcite will expandSearch into an inefficient
+ * string of ORs. The code inside here expands the search into a custom IN operator
+ * when needed.
+ */
+public class ImpalaRexUtil {
+
+ private static RexNode deref(RexProgram program, RexNode node) {
+ while (node instanceof RexLocalRef) {
+ node = requireNonNull(program, "program")
+ .getExprList().get(((RexLocalRef) node).getIndex());
+ }
+ return node;
+ }
+
+ private static <C extends Comparable<C>> RexNode sargRef(RexBuilder rexBuilder,
+ RexNode ref, Sarg<C> sarg, RelDataType type, RexUnknownAs unknownAs) {
+ if (sarg.isAll() || sarg.isNone()) {
+ return RexUtil.simpleSarg(rexBuilder, ref, sarg, unknownAs);
+ }
+ final List<RexNode> orList = new ArrayList<>();
+ if (sarg.nullAs == RexUnknownAs.TRUE
+ && unknownAs == RexUnknownAs.UNKNOWN) {
+ orList.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, ref));
+ }
+ if (sarg.isPoints()) {
+ List<RexNode> operands = new ArrayList<>();
+ operands.add(ref);
+ ///////////////////////
+ // IMPALA change: Generate IN operator
+ ///////////////////////
+ sarg.rangeSet.asRanges().forEach(range ->
+ operands.add(rexBuilder.makeLiteral(range.lowerEndpoint(), type, true, true)));
+ orList.add(rexBuilder.makeCall(ImpalaInOperator.OP, operands));
+ } else if (sarg.isComplementedPoints()) {
+ List<RexNode> operands = new ArrayList<>();
+ operands.add(ref);
+ ///////////////////////
+ // IMPALA change: Generate NOT_IN operator
+ ///////////////////////
+ sarg.rangeSet.complement().asRanges().forEach(range ->
+ operands.add(rexBuilder.makeLiteral(range.lowerEndpoint(), type, true, true)));
+ orList.add(rexBuilder.makeCall(SqlStdOperatorTable.NOT_IN, operands));
+ } else {
+ final RangeSets.Consumer<C> consumer =
+ new RangeToRex<>(ref, orList, rexBuilder, type);
+ RangeSets.forEach(sarg.rangeSet, consumer);
+ }
+ RexNode node = RexUtil.composeDisjunction(rexBuilder, orList);
+ if (sarg.nullAs == RexUnknownAs.FALSE
+ && unknownAs == RexUnknownAs.UNKNOWN) {
+ node =
+ rexBuilder.makeCall(SqlStdOperatorTable.AND,
+ rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, ref),
+ node);
+ }
+ return node;
+ }
+ /** Shuttle that expands calls to
+ * {@link org.apache.calcite.sql.fun.SqlStdOperatorTable#SEARCH}.
+ *
+ * <p>Calls whose complexity is greater than {@link #maxComplexity}
+ * are retained (not expanded). */
+ private static class SearchExpandingShuttle extends RexShuttle {
+ private final RexBuilder rexBuilder;
+ private final RexProgram program;
+ private final int maxComplexity;
+
+ SearchExpandingShuttle(RexProgram program, RexBuilder rexBuilder,
+ int maxComplexity) {
+ this.program = program;
+ this.rexBuilder = rexBuilder;
+ this.maxComplexity = maxComplexity;
+ }
+
+ @Override public RexNode visitCall(RexCall call) {
+ final boolean[] update = {false};
+ final List<RexNode> clonedOperands;
+ switch (call.getKind()) {
+ case SEARCH:
+ final RexNode ref = call.operands.get(0);
+ final RexLiteral literal =
+ (RexLiteral) deref(program, call.operands.get(1));
+ final Sarg sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg");
+ if (maxComplexity < 0 || sarg.complexity() < maxComplexity) {
+ return sargRef(rexBuilder, ref, sarg, literal.getType(),
+ RexUnknownAs.UNKNOWN);
+ }
+ // Sarg is complex (therefore useful); fall through
+ default:
+ return super.visitCall(call);
+ }
+ }
+ }
+
+ /** Converts a {@link Range} to a {@link RexNode} expression.
+ *
+ * @param <C> Value type */
+ private static class RangeToRex<C extends Comparable<C>>
+ implements RangeSets.Consumer<C> {
+ private final List<RexNode> list;
+ private final RexBuilder rexBuilder;
+ private final RelDataType type;
+ private final RexNode ref;
+
+ RangeToRex(RexNode ref, List<RexNode> list, RexBuilder rexBuilder,
+ RelDataType type) {
+ this.ref = requireNonNull(ref, "ref");
+ this.list = requireNonNull(list, "list");
+ this.rexBuilder = requireNonNull(rexBuilder, "rexBuilder");
+ this.type = requireNonNull(type, "type");
+ }
+
+ private void addAnd(RexNode... nodes) {
+ list.add(rexBuilder.makeCall(SqlStdOperatorTable.AND, nodes));
+ }
+
+ private RexNode op(SqlOperator op, C value) {
+ return rexBuilder.makeCall(op, ref,
+ rexBuilder.makeLiteral(value, type, true, true));
+ }
+
+ @Override public void all() {
+ list.add(rexBuilder.makeLiteral(true));
+ }
+
+ @Override public void atLeast(C lower) {
+ list.add(op(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, lower));
+ }
+
+ @Override public void atMost(C upper) {
+ list.add(op(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, upper));
+ }
+
+ @Override public void greaterThan(C lower) {
+ list.add(op(SqlStdOperatorTable.GREATER_THAN, lower));
+ }
+
+ @Override public void lessThan(C upper) {
+ list.add(op(SqlStdOperatorTable.LESS_THAN, upper));
+ }
+
+ @Override public void singleton(C value) {
+ list.add(op(SqlStdOperatorTable.EQUALS, value));
+ }
+
+ @Override public void closed(C lower, C upper) {
+ addAnd(op(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, lower),
+ op(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, upper));
+ }
+
+ @Override public void closedOpen(C lower, C upper) {
+ addAnd(op(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, lower),
+ op(SqlStdOperatorTable.LESS_THAN, upper));
+ }
+
+ @Override public void openClosed(C lower, C upper) {
+ addAnd(op(SqlStdOperatorTable.GREATER_THAN, lower),
+ op(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, upper));
+ }
+
+ @Override public void open(C lower, C upper) {
+ addAnd(op(SqlStdOperatorTable.GREATER_THAN, lower),
+ op(SqlStdOperatorTable.LESS_THAN, upper));
+ }
+ }
+
+ /** Expands calls to {@link SqlStdOperatorTable#SEARCH}
+ * whose complexity is greater than {@code maxComplexity} in an expression. */
+ public static RexNode expandSearch(RexBuilder rexBuilder, RexNode node) {
+ RexShuttle shuttle = new SearchExpandingShuttle(null, rexBuilder, -1);
+ return node.accept(shuttle);
+ }
+
+ // TODO: IMPALA-14432: remove this function when we upgrade to Calcite 1.40
+ // since it is copied from RexBuilder.java
+ public static RexNode makeNullable(RexBuilder rexBuilder, RexNode exp,
+ boolean nullability) {
+ final RelDataType type = exp.getType();
+ if (type.isNullable() == nullability) {
+ return exp;
+ }
+ final RelDataType type2 =
+ rexBuilder.getTypeFactory().createTypeWithNullability(type, nullability);
+ return rexBuilder.makeAbstractCast(type2, exp, false);
+ }
+}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaJoinRel.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaJoinRel.java
index c2d5d8d0c..2c0d255d7 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaJoinRel.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaJoinRel.java
@@ -417,7 +417,8 @@ public class ImpalaJoinRel extends Join
* of the Expr conjuncts (and whether it is an EQUALS conjunct).
*/
private List<ConjunctInfo> getConditionConjuncts(RexNode condition,
- NodeWithExprs leftInput, NodeWithExprs rightInput, Analyzer analyzer) {
+ NodeWithExprs leftInput, NodeWithExprs rightInput, Analyzer analyzer)
+ throws ImpalaException {
List<ConjunctInfo> conjunctInfos = new ArrayList<>();
CreateExprVisitor visitor = new CreateExprVisitor(getCluster().getRexBuilder(),
@@ -429,7 +430,7 @@ public class ImpalaJoinRel extends Join
for (RexNode conj : conjuncts) {
// get a canonicalized representation
conj = getCanonical(conj, leftInput.outputExprs_.size());
- Expr impalaConjunct = conj.accept(visitor);
+ Expr impalaConjunct = CreateExprVisitor.getExpr(visitor, conj);
conjunctInfos.add(
new ConjunctInfo(impalaConjunct, isEquijoinConjunct(conj, leftInput)));
}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/CreateExprVisitor.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/CreateExprVisitor.java
index 447a06056..1cd30d4f2 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/CreateExprVisitor.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/CreateExprVisitor.java
@@ -40,6 +40,7 @@ import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.calcite.functions.RexCallConverter;
import org.apache.impala.calcite.functions.RexLiteralConverter;
+import org.apache.impala.calcite.operators.ImpalaRexUtil;
import java.util.ArrayList;
import java.util.List;
@@ -63,6 +64,10 @@ public class CreateExprVisitor extends RexVisitorImpl<Expr> {
this.analyzer_ = analyzer;
}
+ public RexBuilder getRexBuilder() {
+ return rexBuilder_;
+ }
+
@Override
public Expr visitInputRef(RexInputRef rexInputRef) {
return inputExprs_.get(rexInputRef.getIndex());
@@ -138,7 +143,13 @@ public class CreateExprVisitor extends RexVisitorImpl<Expr> {
public static Expr getExpr(CreateExprVisitor visitor, RexNode operand)
throws ImpalaException {
try {
- Expr expr = operand.accept(visitor);
+ // Impala cannot handle the SEARCH operand so this needs to
+ // be expanded. A custom ImpalaRexUtil was made to handle
+ // the IN operator and is needed until at least CALCITE-7226
+ // is fixed.
+ RexNode expandedOperand =
+ ImpalaRexUtil.expandSearch(visitor.getRexBuilder(), operand);
+ Expr expr = expandedOperand.accept(visitor);
expr.analyze(visitor.analyzer_);
return expr;
} catch (Exception e) {
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/ExprConjunctsConverter.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/ExprConjunctsConverter.java
index 38d99da3c..7df849326 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/ExprConjunctsConverter.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/ExprConjunctsConverter.java
@@ -31,8 +31,10 @@ import org.apache.calcite.rex.RexVisitor;
import org.apache.calcite.sql.SqlKind;
import org.apache.impala.analysis.Analyzer;
import org.apache.impala.analysis.Expr;
+import org.apache.impala.calcite.operators.ImpalaRexUtil;
import org.apache.impala.common.ImpalaException;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -58,7 +60,9 @@ public class ExprConjunctsConverter {
CreateExprVisitor visitor =
new CreateExprVisitor(rexBuilder, inputExprs, analyzer);
- List<RexNode> andOperands = getAndConjuncts(conjunct);
+ RexNode expandedConjunct = ImpalaRexUtil.expandSearch(rexBuilder, conjunct);
+
+ List<RexNode> andOperands = getAndConjuncts(expandedConjunct);
for (RexNode andOperand : andOperands) {
Expr convertedExpr = CreateExprVisitor.getExpr(visitor, andOperand);
builder.add(convertedExpr);
@@ -95,7 +99,11 @@ public class ExprConjunctsConverter {
return ImmutableList.of(conjunct);
}
// If it's an AND conjunct, then all the operands represent individual
- // AND clauses.
- return rexCallConjunct.getOperands();
+ // AND clauses. Call recursively to catch nested ANDs.
+ List<RexNode> andOperands = new ArrayList<>();
+ for (RexNode operand : rexCallConjunct.getOperands()) {
+ andOperands.addAll(getAndConjuncts(operand));
+ }
+ return andOperands;
}
}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaCoreRules.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaCoreRules.java
new file mode 100644
index 000000000..70a5f383f
--- /dev/null
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaCoreRules.java
@@ -0,0 +1,209 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.calcite.rules;
+
+import org.apache.calcite.plan.Contexts;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.rules.AggregateProjectPullUpConstantsRule;
+import org.apache.calcite.rel.rules.FilterJoinRule;
+import org.apache.calcite.rel.rules.FilterAggregateTransposeRule;
+import org.apache.calcite.rel.rules.FilterMergeRule;
+import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
+import org.apache.calcite.rel.rules.FilterSetOpTransposeRule;
+import org.apache.calcite.rel.rules.IntersectToDistinctRule;
+import org.apache.calcite.rel.rules.JoinPushExpressionsRule;
+import org.apache.calcite.rel.rules.JoinPushTransitivePredicatesRule;
+import org.apache.calcite.rel.rules.JoinToMultiJoinRule;
+import org.apache.calcite.rel.rules.ProjectJoinTransposeRule;
+import org.apache.calcite.rel.rules.ProjectMergeRule;
+import org.apache.calcite.rel.rules.SemiJoinRule.ProjectToSemiJoinRule;
+import org.apache.calcite.rel.rules.SortRemoveConstantKeysRule;
+import org.apache.calcite.rel.rules.UnionEliminatorRule;
+import org.apache.calcite.rel.rules.UnionMergeRule;
+import org.apache.calcite.rel.rules.UnionPullUpConstantsRule;
+import org.apache.calcite.rel.rules.UnionToDistinctRule;
+import org.apache.calcite.rel.rules.ValuesReduceRule;
+import org.apache.calcite.rel.rules.SubQueryRemoveRule;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.tools.RelBuilderFactory;
+
+/**
+ * ImpalaCoreRules are the Calcite core rules manipulated to use
+ * an Impala custom RelBuilder which does not call 'simplify' for
+ * the RexNodes. The 'simplify' method rewrites certain rules that
+ * can differ from what Impala needs. See ImpalaRexSimplify for details.
+ */
+
+public class ImpalaCoreRules {
+
+ // The RelBuilder config without the simplify
+ // TODO: Ideally, there should be an interface within RelBuilder
+ // which allows our own RexSimplify to be used. Once this happens,
+ // we can add our own RexSimplify to RelBuilder and remove all
+ // the rules here that use this NO_SIMPLIFY configuration.
+ private static final RelBuilder.Config CONFIG_NO_SIMPLIFY =
+ RelBuilder.Config.DEFAULT
+ .withSimplify(false);
+
+ public static final RelBuilderFactory LOGICAL_BUILDER_NO_SIMPLIFY =
+ RelBuilder.proto(Contexts.of(RelFactories.DEFAULT_STRUCT,
+ CONFIG_NO_SIMPLIFY));
+
+ public static RelOptRule PROJECT_SUB_QUERY_TO_CORRELATE =
+ SubQueryRemoveRule.Config.PROJECT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(SubQueryRemoveRule.Config.class).toRule();
+
+ public static RelOptRule FILTER_SUB_QUERY_TO_CORRELATE =
+ SubQueryRemoveRule.Config.FILTER
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(SubQueryRemoveRule.Config.class).toRule();
+
+ public static RelOptRule JOIN_SUB_QUERY_TO_CORRELATE =
+ SubQueryRemoveRule.Config.JOIN
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(SubQueryRemoveRule.Config.class).toRule();
+
+ // Rules from Calcite modified to avoid using simplify.
+ // Simplify will be run through its own rule with a custom
+ // derived RexSimplify
+
+ public static JoinPushTransitivePredicatesRule JOIN_PUSH_TRANSITIVE_PREDICATES =
+ JoinPushTransitivePredicatesRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(JoinPushTransitivePredicatesRule.Config.class).toRule();
+
+ public static FilterJoinRule JOIN_CONDITION_PUSH =
+ FilterJoinRule.JoinConditionPushRule.JoinConditionPushRuleConfig.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(FilterJoinRule.JoinConditionPushRule.
+ JoinConditionPushRuleConfig.class).toRule();
+
+ public static JoinPushExpressionsRule JOIN_PUSH_EXPRESSIONS =
+ JoinPushExpressionsRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(JoinPushExpressionsRule.Config.class).toRule();
+
+ public static FilterJoinRule FILTER_INTO_JOIN =
+ FilterJoinRule.FilterIntoJoinRule.FilterIntoJoinRuleConfig.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(FilterJoinRule.FilterIntoJoinRule.
+ FilterIntoJoinRuleConfig.class).toRule();
+
+ public static IntersectToDistinctRule INTERSECT_TO_DISTINCT =
+ IntersectToDistinctRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(IntersectToDistinctRule.Config.class).toRule();
+
+ public static UnionToDistinctRule UNION_TO_DISTINCT =
+ UnionToDistinctRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(UnionToDistinctRule.Config.class).toRule();
+
+ public static UnionMergeRule UNION_MERGE =
+ UnionMergeRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(UnionMergeRule.Config.class).toRule();
+
+ public static UnionPullUpConstantsRule UNION_PULL_UP_CONSTANTS =
+ UnionPullUpConstantsRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(UnionPullUpConstantsRule.Config.class).toRule();
+
+ public static AggregateProjectPullUpConstantsRule AGGREGATE_ANY_PULL_UP_CONSTANTS =
+ AggregateProjectPullUpConstantsRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(AggregateProjectPullUpConstantsRule.Config.class).toRule();
+
+ public static FilterProjectTransposeRule FILTER_PROJECT_TRANSPOSE =
+ FilterProjectTransposeRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(FilterProjectTransposeRule.Config.class).toRule();
+
+ public static FilterSetOpTransposeRule FILTER_SET_OP_TRANSPOSE =
+ FilterSetOpTransposeRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(FilterSetOpTransposeRule.Config.class).toRule();
+
+ public static FilterAggregateTransposeRule FILTER_AGGREGATE_TRANSPOSE =
+ FilterAggregateTransposeRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(FilterAggregateTransposeRule.Config.class).toRule();
+
+ public static ProjectJoinTransposeRule PROJECT_JOIN_TRANSPOSE =
+ ProjectJoinTransposeRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(ProjectJoinTransposeRule.Config.class).toRule();
+
+ public static UnionEliminatorRule UNION_REMOVE =
+ UnionEliminatorRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(UnionEliminatorRule.Config.class).toRule();
+
+ public static ProjectToSemiJoinRule PROJECT_TO_SEMI_JOIN =
+ ProjectToSemiJoinRule.ProjectToSemiJoinRuleConfig.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(ProjectToSemiJoinRule.ProjectToSemiJoinRuleConfig.class).toRule();
+
+ public static ValuesReduceRule FILTER_VALUES_MERGE =
+ ValuesReduceRule.Config.FILTER
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(ValuesReduceRule.Config.class).toRule();
+
+ public static FilterMergeRule FILTER_MERGE =
+ FilterMergeRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(FilterMergeRule.Config.class).toRule();
+
+ public static ProjectMergeRule PROJECT_MERGE =
+ ProjectMergeRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(ProjectMergeRule.Config.class).toRule();
+
+ public static JoinToMultiJoinRule JOIN_TO_MULTI_JOIN =
+ JoinToMultiJoinRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(JoinToMultiJoinRule.Config.class).toRule();
+
+ public static SortRemoveConstantKeysRule SORT_REMOVE_CONSTANT_KEYS =
+ SortRemoveConstantKeysRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(SortRemoveConstantKeysRule.Config.class).toRule();
+
+ // Impala defined rules
+ public static RelOptRule IMPALA_MINUS_TO_DISTINCT =
+ ImpalaMinusToDistinctRule.Config.DEFAULT
+ .withRelBuilderFactory(LOGICAL_BUILDER_NO_SIMPLIFY)
+ .as(ImpalaMinusToDistinctRule.Config.class).toRule();
+
+ public static RelOptRule COMBINE_VALUES_NODES =
+ new CombineValuesNodesRule();
+
+ public static RelOptRule EXTRACT_LITERAL_AGG =
+ new ExtractLiteralAgg();
+
+ public static RelOptRule REWRITE_REX_OVER = RewriteRexOverRule.INSTANCE;
+
+ public static RelOptRule JOIN_PROJECT_TRANSPOSE_LEFT_OUTER =
+ ImpalaJoinProjectTransposeRule.LEFT_OUTER;
+
+ public static RelOptRule JOIN_PROJECT_TRANSPOSE_RIGHT_OUTER =
+ ImpalaJoinProjectTransposeRule.RIGHT_OUTER;
+
+}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaFilterSimplifyRule.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaFilterSimplifyRule.java
new file mode 100644
index 000000000..2063af509
--- /dev/null
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaFilterSimplifyRule.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.impala.calcite.rules;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.impala.calcite.operators.ImpalaRexSimplify;
+import org.apache.impala.calcite.operators.ImpalaRexUtil;
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * ImpalaFilterSimplifyRule calls the given ImpalaRexSimplify.simplify()
+ * method (derived from Calcite's RexSimplify) for the filter condition.
+ */
+public class ImpalaFilterSimplifyRule extends RelOptRule {
+
+ private final ImpalaRexSimplify simplifier_;
+
+ public ImpalaFilterSimplifyRule(ImpalaRexSimplify simplifier) {
+ super(operand(Filter.class, none()));
+ this.simplifier_ = simplifier;
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ Filter filter = call.rel(0);
+ RelOptCluster cluster = filter.getCluster();
+ RexBuilder rexBuilder = cluster.getRexBuilder();
+ RexNode condition = filter.getCondition();
+
+ RexNode newCondition = simplifier_.simplify(condition);
+
+ if (newCondition.equals(condition)) {
+ return;
+ }
+
+ Filter newFilter =
+ filter.copy(filter.getTraitSet(), filter.getInput(0), newCondition);
+ call.transformTo(newFilter);
+ }
+}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaProjectSimplifyRule.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaProjectSimplifyRule.java
new file mode 100644
index 000000000..b6d8d2adb
--- /dev/null
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaProjectSimplifyRule.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.impala.calcite.rules;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.impala.calcite.operators.ImpalaRexSimplify;
+import org.apache.impala.calcite.operators.ImpalaRexUtil;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * ImpalaProjectSimplifyRule calls the given ImpalaRexSimplify.simplify()
+ * method (derived from Calcite's RexSimplify) for all the columns in the
+ * Project RelNode.
+ */
+public class ImpalaProjectSimplifyRule extends RelOptRule {
+
+ private final ImpalaRexSimplify simplifier_;
+
+ public ImpalaProjectSimplifyRule(ImpalaRexSimplify simplifier) {
+ super(operand(Project.class, none()));
+ this.simplifier_ = simplifier;
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ Project projectRel = call.rel(0);
+ RelOptCluster cluster = projectRel.getCluster();
+ RexBuilder rexBuilder = cluster.getRexBuilder();
+
+ List<RexNode> newProjects = new ArrayList<>();
+ boolean projectChanged = false;
+ for (RexNode project : projectRel.getProjects()) {
+ RexNode newProject = simplifier_.simplify(project);
+
+ // Need to make sure the nullables match for projects so the parent row
+ // type gets the nullability it expects.
+ if (newProject.getType().isNullable() != project.getType().isNullable()) {
+ newProject = ImpalaRexUtil.makeNullable(rexBuilder, newProject,
+ project.getType().isNullable());
+ }
+ if (!newProject.equals(project)) {
+ projectChanged = true;
+ }
+ newProjects.add(newProject);
+ }
+
+ if (!projectChanged) {
+ return;
+ }
+
+ // only create the new project if any projects changed.
+ Project newProjectRel = projectRel.copy(projectRel.getTraitSet(),
+ projectRel.getInput(0), newProjects, projectRel.getRowType());
+ call.transformTo(newProjectRel);
+ }
+}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/FilterSelectivityEstimator.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/FilterSelectivityEstimator.java
index 9f751e263..6bc8d3a42 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/FilterSelectivityEstimator.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/FilterSelectivityEstimator.java
@@ -26,10 +26,12 @@ import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Sarg;
import org.apache.impala.calcite.rel.util.RexInputRefCollector;
import org.apache.impala.calcite.schema.JoinRelationInfo.EqualityConjunction;
import org.apache.impala.catalog.Column;
@@ -136,8 +138,8 @@ public class FilterSelectivityEstimator {
case BETWEEN:
// TODO: Impala has better logic than this
return BETWEEN_SELECTIVITY;
- case IN:
- return computeInSelectivity(call);
+ case SEARCH:
+ return computeSearchSelectivity(call);
default:
return computeEqualsSelectivity(call);
}
@@ -206,10 +208,26 @@ public class FilterSelectivityEstimator {
return computeNotEqualitySelectivity(call);
}
- private Double computeInSelectivity(RexCall call) {
- Double selectivity = computeEqualsSelectivity(call);
- selectivity = selectivity * (call.operands.size() - 1);
- return Math.min(selectivity, 1.0);
+ private Double computeSearchSelectivity(RexCall call) {
+ try {
+ RexLiteral literal = (RexLiteral) call.getOperands().get(1);
+ Sarg<?> sarg = literal.getValueAs(Sarg.class);
+ if (sarg.isPoints() || sarg.isComplementedPoints()) {
+ Double selectivity = computeEqualsSelectivity(call);
+ selectivity = selectivity * sarg.pointCount;
+ return sarg.isPoints()
+ ? Math.min(selectivity, 1.0)
+ : Math.max(1.0 - selectivity, 0.0);
+ } else {
+ // TODO: Impala has better logic than this
+ return BETWEEN_SELECTIVITY;
+ }
+ } catch (Exception e) {
+ LOG.warn("Warning: Bug found when trying to calculate selectivity for search " +
+ "operator, but instead of throwing an exception, a default selectivity will " +
+ "be used.");
+ return computeEqualsSelectivity(call);
+ }
}
/**
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java
index 53b960192..f09376ebe 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java
@@ -34,19 +34,19 @@ import org.apache.calcite.rel.rules.CoreRules;
import org.apache.calcite.rel.rules.JoinProjectTransposeRule;
import org.apache.calcite.rel.rules.PruneEmptyRules;
import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.SqlExplainFormat;
import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql2rel.RelFieldTrimmer;
import org.apache.calcite.tools.RelBuilder;
import org.apache.impala.calcite.coercenodes.CoerceNodes;
+import org.apache.impala.calcite.operators.ImpalaRexSimplify;
import org.apache.impala.calcite.rel.node.ConvertToImpalaRelRules;
import org.apache.impala.calcite.rel.node.ImpalaPlanRel;
-import org.apache.impala.calcite.rules.CombineValuesNodesRule;
-import org.apache.impala.calcite.rules.ExtractLiteralAgg;
-import org.apache.impala.calcite.rules.ImpalaJoinProjectTransposeRule;
-import org.apache.impala.calcite.rules.ImpalaMinusToDistinctRule;
-import org.apache.impala.calcite.rules.RewriteRexOverRule;
+import org.apache.impala.calcite.rules.ImpalaCoreRules;
+import org.apache.impala.calcite.rules.ImpalaFilterSimplifyRule;
+import org.apache.impala.calcite.rules.ImpalaProjectSimplifyRule;
import org.apache.impala.calcite.util.LogUtil;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.util.EventSequence;
@@ -104,15 +104,17 @@ public class CalciteOptimizer implements CompilerStep {
}
public ImpalaPlanRel optimize(RelNode logPlan) throws ImpalaException {
- RelBuilder relBuilder = RelFactories.LOGICAL_BUILDER.create(logPlan.getCluster(),
- reader_);
+ RelBuilder relBuilder = ImpalaCoreRules.LOGICAL_BUILDER_NO_SIMPLIFY.create(
+ logPlan.getCluster(), reader_);
RexBuilder rexBuilder = logPlan.getCluster().getRexBuilder();
+ ImpalaRexSimplify simplifier = new ImpalaRexSimplify(rexBuilder, RexUtil.EXECUTOR);
+
// Run some essential rules needed to create working RelNodes before doing
// optimization
timeline_.markEvent("Starting optimization");
- RelNode expandedNodesPlan = runExpandNodesProgram(logPlan);
+ RelNode expandedNodesPlan = runExpandNodesProgram(logPlan, simplifier);
timeline_.markEvent("Expanded plan");
LogUtil.logDebug(expandedNodesPlan, "Plan after expanded plan phase.");
@@ -127,20 +129,21 @@ public class CalciteOptimizer implements CompilerStep {
LogUtil.logDebug(coercedNodesPlan, "Plan after it has been coerced.");
// Run rules that swap RelNodes and optimize the expressions within a RelNode
- RelNode preJoinOptimizedPlan = runOptimizeNodesProgram(relBuilder, coercedNodesPlan);
+ RelNode preJoinOptimizedPlan = runOptimizeNodesProgram(relBuilder, rexBuilder,
+ coercedNodesPlan, simplifier);
timeline_.markEvent("Created optimized plan pre join");
LogUtil.logDebug(preJoinOptimizedPlan, "Optimized plan before join rules " +
"have been applied.");
// Run join optimization
- RelNode optimizedJoinPlan = runJoinProgram(preJoinOptimizedPlan);
+ RelNode optimizedJoinPlan = runJoinProgram(preJoinOptimizedPlan, simplifier);
timeline_.markEvent("Created optimized join plan");
LogUtil.logDebug(optimizedJoinPlan, "Optimized plan after join optimization.");
// rerun rules that swap RelNodes and optimize the expressions within a RelNode,
// since the join optimization may have enabled some more rules that can be applied.
RelNode postOptimizedJoinPlan =
- runOptimizeNodesProgram(relBuilder, optimizedJoinPlan);
+ runOptimizeNodesProgram(relBuilder, rexBuilder, optimizedJoinPlan, simplifier);
timeline_.markEvent("Created optimized plan post join");
LogUtil.logDebug(postOptimizedJoinPlan, "Optimized plan after a second pass of "
+ "rules applied after join optimization.");
@@ -148,41 +151,42 @@ public class CalciteOptimizer implements CompilerStep {
// Run some essential rules needed to create working RelNodes after
// optimization
RelNode preImpalaConvertPlan =
- runPreImpalaConvertProgram(postOptimizedJoinPlan);
+ runPreImpalaConvertProgram(postOptimizedJoinPlan, simplifier);
LogUtil.logDebug(preImpalaConvertPlan, "Optimized plan after final "
+ "preparation done before conversion to physical nodes.");
// Change the Calcite RelNodes into ImpalaPlanRel RelNodes, all of which
// contain a method that converts the RelNodes into Impala PlanNodes.
ImpalaPlanRel finalOptimizedPlan =
- runImpalaConvertProgram(preImpalaConvertPlan);
+ runImpalaConvertProgram(preImpalaConvertPlan, simplifier);
timeline_.markEvent("Created final Impala convert plan");
LogUtil.logDebug(finalOptimizedPlan, "Final Impala optimized plan");
return finalOptimizedPlan;
}
- private RelNode runExpandNodesProgram(RelNode plan) throws ImpalaException {
+ private RelNode runExpandNodesProgram(RelNode plan,
+ ImpalaRexSimplify simplifier) throws ImpalaException {
HepProgramBuilder builder = new HepProgramBuilder();
builder.addMatchOrder(HepMatchOrder.BOTTOM_UP);
builder.addRuleCollection(ImmutableList.of(
- CoreRules.INTERSECT_TO_DISTINCT,
- CoreRules.UNION_TO_DISTINCT,
- ImpalaMinusToDistinctRule.Config.DEFAULT.toRule(),
- new CombineValuesNodesRule(),
- new ExtractLiteralAgg(),
- CoreRules.SORT_REMOVE_CONSTANT_KEYS
+ ImpalaCoreRules.INTERSECT_TO_DISTINCT,
+ ImpalaCoreRules.UNION_TO_DISTINCT,
+ ImpalaCoreRules.IMPALA_MINUS_TO_DISTINCT,
+ ImpalaCoreRules.COMBINE_VALUES_NODES,
+ ImpalaCoreRules.EXTRACT_LITERAL_AGG,
+ ImpalaCoreRules.SORT_REMOVE_CONSTANT_KEYS
));
builder.addMatchOrder(HepMatchOrder.BOTTOM_UP);
- return runProgram(plan, builder.build());
+ return runProgram(plan, builder.build(), simplifier);
}
- private RelNode runOptimizeNodesProgram(RelBuilder relBuilder,
- RelNode plan) throws ImpalaException {
+ private RelNode runOptimizeNodesProgram(RelBuilder relBuilder, RexBuilder rexBuilder,
+ RelNode plan, ImpalaRexSimplify simplifier) throws ImpalaException {
RelFieldTrimmer trimmer =
new RelFieldTrimmer(validator_, relBuilder);
@@ -191,19 +195,21 @@ public class CalciteOptimizer implements CompilerStep {
HepProgramBuilder builder = new HepProgramBuilder();
List<RelOptRule> interRules = ImmutableList.of(
- CoreRules.UNION_PULL_UP_CONSTANTS,
- CoreRules.AGGREGATE_ANY_PULL_UP_CONSTANTS,
- CoreRules.FILTER_PROJECT_TRANSPOSE,
- CoreRules.FILTER_SET_OP_TRANSPOSE,
- CoreRules.JOIN_CONDITION_PUSH,
- CoreRules.FILTER_INTO_JOIN,
- CoreRules.FILTER_AGGREGATE_TRANSPOSE,
- CoreRules.PROJECT_JOIN_TRANSPOSE,
- CoreRules.UNION_REMOVE,
- CoreRules.PROJECT_TO_SEMI_JOIN,
- CoreRules.FILTER_VALUES_MERGE,
- CoreRules.FILTER_MERGE,
- CoreRules.PROJECT_MERGE,
+ new ImpalaFilterSimplifyRule(simplifier),
+ new ImpalaProjectSimplifyRule(simplifier),
+ ImpalaCoreRules.UNION_PULL_UP_CONSTANTS,
+ ImpalaCoreRules.AGGREGATE_ANY_PULL_UP_CONSTANTS,
+ ImpalaCoreRules.FILTER_PROJECT_TRANSPOSE,
+ ImpalaCoreRules.FILTER_SET_OP_TRANSPOSE,
+ ImpalaCoreRules.JOIN_CONDITION_PUSH,
+ ImpalaCoreRules.FILTER_INTO_JOIN,
+ ImpalaCoreRules.FILTER_AGGREGATE_TRANSPOSE,
+ ImpalaCoreRules.UNION_REMOVE,
+ ImpalaCoreRules.PROJECT_TO_SEMI_JOIN,
+ ImpalaCoreRules.FILTER_VALUES_MERGE,
+ ImpalaCoreRules.FILTER_MERGE,
+ ImpalaCoreRules.PROJECT_MERGE,
+ ImpalaCoreRules.JOIN_PUSH_EXPRESSIONS,
PruneEmptyRules.PROJECT_INSTANCE,
PruneEmptyRules.AGGREGATE_INSTANCE,
PruneEmptyRules.SORT_INSTANCE,
@@ -215,18 +221,20 @@ public class CalciteOptimizer implements CompilerStep {
builder.addMatchOrder(HepMatchOrder.TOP_DOWN);
builder.addRuleCollection(interRules);
- return trimmer.trim(runProgram(plan, builder.build()));
+ return trimmer.trim(runProgram(trimmedPlan, builder.build(), simplifier));
}
/**
* Run the rules specifically for join ordering.
*
*/
- private RelNode runJoinProgram(RelNode plan) throws ImpalaException {
+ private RelNode runJoinProgram(RelNode plan,
+ ImpalaRexSimplify simplifier) throws ImpalaException {
HepProgramBuilder builder = new HepProgramBuilder();
// has to be in a separate program or else there is an infinite loop
- builder.addRuleInstance(CoreRules.JOIN_PUSH_TRANSITIVE_PREDICATES);
+ builder.addRuleInstance(ImpalaCoreRules.JOIN_PUSH_TRANSITIVE_PREDICATES);
+ builder.addRuleInstance(CoreRules.JOIN_DERIVE_IS_NOT_NULL_FILTER_RULE);
// XXX: add comment about project
// Merge the filter nodes into the Join. Also include
@@ -236,11 +244,11 @@ public class CalciteOptimizer implements CompilerStep {
// joins next to each other if possible for the join
// optimization step.
builder.addRuleCollection(ImmutableList.of(
- ImpalaJoinProjectTransposeRule.LEFT_OUTER,
- ImpalaJoinProjectTransposeRule.RIGHT_OUTER,
- CoreRules.FILTER_PROJECT_TRANSPOSE,
- CoreRules.PROJECT_MERGE,
- CoreRules.FILTER_INTO_JOIN
+ ImpalaCoreRules.JOIN_PROJECT_TRANSPOSE_LEFT_OUTER,
+ ImpalaCoreRules.JOIN_PROJECT_TRANSPOSE_RIGHT_OUTER,
+ ImpalaCoreRules.FILTER_PROJECT_TRANSPOSE,
+ ImpalaCoreRules.PROJECT_MERGE,
+ ImpalaCoreRules.FILTER_INTO_JOIN
));
// Join rules work in a two step process. The first step
@@ -249,11 +257,11 @@ public class CalciteOptimizer implements CompilerStep {
// MULTI_JOIN_OPTIMIZE rule is used to determine the join
// ordering.
builder.addMatchOrder(HepMatchOrder.BOTTOM_UP);
- builder.addRuleInstance(CoreRules.JOIN_CONDITION_PUSH);
- builder.addRuleInstance(CoreRules.JOIN_TO_MULTI_JOIN);
+ builder.addRuleInstance(ImpalaCoreRules.JOIN_CONDITION_PUSH);
+ builder.addRuleInstance(ImpalaCoreRules.JOIN_TO_MULTI_JOIN);
builder.addRuleInstance(CoreRules.MULTI_JOIN_OPTIMIZE);
- return runProgram(plan, builder.build());
+ return runProgram(plan, builder.build(), simplifier);
}
/**
@@ -267,22 +275,24 @@ public class CalciteOptimizer implements CompilerStep {
* FILTER_PROJECT_TRANSPOSE: One last transpose is done since the physical
* conversion needs the Logical RelNodes ordered in this way.
*/
- private RelNode runPreImpalaConvertProgram(RelNode plan) throws ImpalaException {
+ private RelNode runPreImpalaConvertProgram(RelNode plan,
+ ImpalaRexSimplify simplifier) throws ImpalaException {
HepProgramBuilder builder = new HepProgramBuilder();
RelNode retRelNode = plan;
builder.addMatchOrder(HepMatchOrder.BOTTOM_UP);
builder.addRuleCollection(ImmutableList.of(
- RewriteRexOverRule.INSTANCE,
- CoreRules.FILTER_PROJECT_TRANSPOSE
+ ImpalaCoreRules.REWRITE_REX_OVER,
+ ImpalaCoreRules.FILTER_PROJECT_TRANSPOSE
));
- return runProgram(retRelNode, builder.build());
+ return runProgram(retRelNode, builder.build(), simplifier);
}
- private ImpalaPlanRel runImpalaConvertProgram(RelNode plan) throws ImpalaException {
+ private ImpalaPlanRel runImpalaConvertProgram(RelNode plan,
+ ImpalaRexSimplify simplifier) throws ImpalaException {
HepProgramBuilder builder = new HepProgramBuilder();
builder.addRuleCollection(ImmutableList.of(
@@ -296,14 +306,16 @@ public class CalciteOptimizer implements CompilerStep {
new ConvertToImpalaRelRules.ImpalaValuesRule()
));
- return (ImpalaPlanRel) runProgram(plan, builder.build());
+ return (ImpalaPlanRel) runProgram(plan, builder.build(), simplifier);
}
- private RelNode runProgram(RelNode currentNode, HepProgram program) {
+ private RelNode runProgram(RelNode currentNode, HepProgram program,
+ ImpalaRexSimplify simplifier) {
HepPlanner planner = new HepPlanner(program,
currentNode.getCluster().getPlanner().getContext(), true, null,
RelOptCostImpl.FACTORY);
planner.setRoot(currentNode);
+ planner.setExecutor(simplifier.getRexExecutor());
return planner.findBestExp();
}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteRelNodeConverter.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteRelNodeConverter.java
index 72f693c92..d932b821b 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteRelNodeConverter.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteRelNodeConverter.java
@@ -26,7 +26,6 @@ import org.apache.calcite.plan.RelOptCostImpl;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.hep.HepMatchOrder;
import org.apache.calcite.plan.hep.HepPlanner;
import org.apache.calcite.plan.hep.HepProgramBuilder;
@@ -35,15 +34,11 @@ import org.apache.calcite.prepare.CalciteCatalogReader;
import org.apache.calcite.prepare.PlannerImpl;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelRoot;
-import org.apache.calcite.rel.core.RelFactories;
-import org.apache.calcite.rel.rules.CoreRules;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.SqlBasicCall;
-import org.apache.calcite.sql.SqlExplainFormat;
-import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlSelect;
@@ -53,12 +48,11 @@ import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.calcite.sql2rel.RelDecorrelator;
import org.apache.calcite.sql2rel.SqlToRelConverter;
-import org.apache.calcite.sql2rel.StandardConvertletTable;
import org.apache.calcite.tools.FrameworkConfig;
import org.apache.calcite.tools.Frameworks;
import org.apache.calcite.tools.RelBuilder;
import org.apache.impala.calcite.operators.ImpalaConvertletTable;
-import org.apache.impala.calcite.operators.ImpalaRexBuilder;
+import org.apache.impala.calcite.rules.ImpalaCoreRules;
import org.apache.impala.calcite.schema.ImpalaRelMetadataProvider;
import org.apache.impala.calcite.util.LogUtil;
@@ -94,7 +88,7 @@ public class CalciteRelNodeConverter implements CompilerStep {
this.planner_ = new VolcanoPlanner();
planner_.addRelTraitDef(ConventionTraitDef.INSTANCE);
cluster_ =
- RelOptCluster.create(planner_, new ImpalaRexBuilder(typeFactory_));
+ RelOptCluster.create(planner_, new RexBuilder(typeFactory_));
viewExpander_ = createViewExpander(
analysisResult.getSqlValidator().getCatalogReader().getRootSchema().plus());
cluster_.setMetadataProvider(ImpalaRelMetadataProvider.DEFAULT);
@@ -107,7 +101,7 @@ public class CalciteRelNodeConverter implements CompilerStep {
this.planner_ = new VolcanoPlanner();
planner_.addRelTraitDef(ConventionTraitDef.INSTANCE);
cluster_ =
- RelOptCluster.create(planner_, new ImpalaRexBuilder(typeFactory_));
+ RelOptCluster.create(planner_, new RexBuilder(typeFactory_));
viewExpander_ = createViewExpander(validator.getCatalogReader()
.getRootSchema().plus());
cluster_.setMetadataProvider(ImpalaRelMetadataProvider.DEFAULT);
@@ -134,13 +128,20 @@ public class CalciteRelNodeConverter implements CompilerStep {
}
public RelNode convert(SqlNode validatedNode) {
+ // Use the NO_SIMPLIFY RelBuilderFactory. Starting around Calcite 1.40, there
+ // are cases where Calcite finds a common type for literal strings that do not
+ // have the same length to the higher CHAR type. Impala treats literal strings
+ // as STRING type. The simplify() method removes some vital information needed
+ // to convert the CHAR to a STRING type later in coerce nodes, so we avoid the
+ // simplify step until after coerce nodes is complete.
SqlToRelConverter relConverter = new SqlToRelConverter(
viewExpander_,
sqlValidator_,
reader_,
cluster_,
ImpalaConvertletTable.INSTANCE,
- SqlToRelConverter.config().withCreateValuesRel(false));
+ SqlToRelConverter.config().withCreateValuesRel(false)
+ .withRelBuilderFactory(ImpalaCoreRules.LOGICAL_BUILDER_NO_SIMPLIFY));
// Convert the valid AST into a logical plan
RelRoot root = relConverter.convertQuery(validatedNode, false, true);
@@ -150,15 +151,16 @@ public class CalciteRelNodeConverter implements CompilerStep {
RelNode subQueryRemovedPlan =
runProgram(
ImmutableList.of(
- CoreRules.JOIN_SUB_QUERY_TO_CORRELATE,
- CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE,
- CoreRules.FILTER_SUB_QUERY_TO_CORRELATE
+ ImpalaCoreRules.JOIN_SUB_QUERY_TO_CORRELATE,
+ ImpalaCoreRules.PROJECT_SUB_QUERY_TO_CORRELATE,
+ ImpalaCoreRules.FILTER_SUB_QUERY_TO_CORRELATE
),
relNode);
LogUtil.logDebug(subQueryRemovedPlan, "Plan after subquery removal phase");
- RelBuilder relBuilder = RelFactories.LOGICAL_BUILDER.create(cluster_,
- reader_);
+ RelBuilder relBuilder =
+ ImpalaCoreRules.LOGICAL_BUILDER_NO_SIMPLIFY.create(cluster_, reader_);
+
RelNode decorrelatedPlan =
RelDecorrelator.decorrelateQuery(subQueryRemovedPlan, relBuilder);
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/type/ImpalaTypeConverter.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/type/ImpalaTypeConverter.java
index c4139930c..49594cff3 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/type/ImpalaTypeConverter.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/type/ImpalaTypeConverter.java
@@ -65,6 +65,8 @@ public class ImpalaTypeConverter {
// Maps Impala default types to Calcite default types.
private static Map<Type, RelDataType> impalaToCalciteMap;
+ private static Map<Type, RelDataType> nonNullImpalaToCalciteMap;
+
static {
RexBuilder rexBuilder =
new RexBuilder(new JavaTypeFactoryImpl(new ImpalaTypeSystemImpl()));
@@ -87,10 +89,13 @@ public class ImpalaTypeConverter {
map.put(Type.NULL, factory.createSqlType(SqlTypeName.NULL));
ImmutableMap.Builder<Type, RelDataType> builder = ImmutableMap.builder();
+ ImmutableMap.Builder<Type, RelDataType> nonNullBuilder = ImmutableMap.builder();
for (Type t : map.keySet()) {
RelDataType r = map.get(t);
+ nonNullBuilder.put(t, r);
builder.put(t, factory.createTypeWithNullability(r, true));
}
+ nonNullImpalaToCalciteMap = nonNullBuilder.build();
impalaToCalciteMap = builder.build();
}
@@ -127,12 +132,21 @@ public class ImpalaTypeConverter {
* Get the normalized RelDataType given an impala type.
*/
public static RelDataType getRelDataType(Type impalaType) {
+ return getRelDataType(impalaType, true);
+ }
+
+ /**
+ * Get the normalized RelDataType given an impala type.
+ */
+ public static RelDataType getRelDataType(Type impalaType, boolean nullable) {
if (impalaType == null) {
return null;
}
TPrimitiveType primitiveType = impalaType.getPrimitiveType().toThrift();
Type normalizedImpalaType = getImpalaType(primitiveType);
- return impalaToCalciteMap.get(normalizedImpalaType);
+ return nullable
+ ? impalaToCalciteMap.get(normalizedImpalaType)
+ : nonNullImpalaToCalciteMap.get(normalizedImpalaType);
}
/**
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/calcite/analytic-rank-pushdown-calcite.test b/testdata/workloads/functional-planner/queries/PlannerTest/calcite/analytic-rank-pushdown-calcite.test
index 82add1381..24dc0c210 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/calcite/analytic-rank-pushdown-calcite.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/calcite/analytic-rank-pushdown-calcite.test
@@ -8,7 +8,7 @@ where rnk < 100 and int_col is not null
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 100
+| predicates: int_col IS NOT NULL, row_number() < 100
| row-size=31B cardinality=10
|
02:ANALYTIC
@@ -29,7 +29,7 @@ PLAN-ROOT SINK
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 100
+| predicates: int_col IS NOT NULL, row_number() < 100
| row-size=31B cardinality=10
|
02:ANALYTIC
@@ -176,7 +176,7 @@ where rnk < 100 and int_col is not null
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 100
+| predicates: int_col IS NOT NULL, row_number() < 100
| row-size=31B cardinality=10
|
02:ANALYTIC
@@ -197,7 +197,7 @@ PLAN-ROOT SINK
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 100
+| predicates: int_col IS NOT NULL, row_number() < 100
| row-size=31B cardinality=10
|
02:ANALYTIC
@@ -230,7 +230,7 @@ where rnk < 100 and int_col is not null
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), rank() < 100
+| predicates: int_col IS NOT NULL, rank() < 100
| row-size=31B cardinality=10
|
02:ANALYTIC
@@ -252,7 +252,7 @@ PLAN-ROOT SINK
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), rank() < 100
+| predicates: int_col IS NOT NULL, rank() < 100
| row-size=31B cardinality=10
|
02:ANALYTIC
@@ -288,7 +288,7 @@ where rnk < 5 and int_col is not null
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 5
+| predicates: int_col IS NOT NULL, row_number() < 5
| row-size=31B cardinality=383
|
02:ANALYTIC
@@ -314,7 +314,7 @@ PLAN-ROOT SINK
06:EXCHANGE [UNPARTITIONED]
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 5
+| predicates: int_col IS NOT NULL, row_number() < 5
| row-size=31B cardinality=383
|
02:ANALYTIC
@@ -353,7 +353,7 @@ where rnk < 5 and int_col is not null
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 5
+| predicates: int_col IS NOT NULL, row_number() < 5
| row-size=31B cardinality=383
|
02:ANALYTIC
@@ -379,7 +379,7 @@ PLAN-ROOT SINK
06:EXCHANGE [UNPARTITIONED]
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 5
+| predicates: int_col IS NOT NULL, row_number() < 5
| row-size=31B cardinality=383
|
02:ANALYTIC
@@ -517,7 +517,7 @@ where 99 >= rnk and int_col is not null
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() <= 99
+| predicates: int_col IS NOT NULL, row_number() <= 99
| row-size=31B cardinality=10
|
02:ANALYTIC
@@ -538,7 +538,7 @@ PLAN-ROOT SINK
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() <= 99
+| predicates: int_col IS NOT NULL, row_number() <= 99
| row-size=31B cardinality=10
|
02:ANALYTIC
@@ -570,7 +570,7 @@ where rnk < 5 and int_col is not null
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 5
+| predicates: int_col IS NOT NULL, row_number() < 5
| row-size=32B cardinality=1
|
02:ANALYTIC
@@ -596,7 +596,7 @@ PLAN-ROOT SINK
06:EXCHANGE [UNPARTITIONED]
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 5
+| predicates: int_col IS NOT NULL, row_number() < 5
| row-size=32B cardinality=1
|
02:ANALYTIC
@@ -635,7 +635,7 @@ where rnk < 100 and int_col is not null
PLAN-ROOT SINK
|
03:SELECT
-| predicates: dense_rank() < 100, is_not_null_pred(int_col)
+| predicates: dense_rank() < 100, int_col IS NOT NULL
| row-size=31B cardinality=1.10K
|
02:ANALYTIC
@@ -666,7 +666,7 @@ where rnk < 5 and int_col is not null
PLAN-ROOT SINK
|
05:SELECT
-| predicates: is_not_null_pred(first_value(functional.alltypes.int_col)), row_number() < 5
+| predicates: first_value(functional.alltypes.int_col) IS NOT NULL, row_number() < 5
| row-size=33B cardinality=4
|
04:ANALYTIC
@@ -702,7 +702,7 @@ PLAN-ROOT SINK
09:EXCHANGE [UNPARTITIONED]
|
05:SELECT
-| predicates: is_not_null_pred(first_value(functional.alltypes.int_col)), row_number() < 5
+| predicates: first_value(functional.alltypes.int_col) IS NOT NULL, row_number() < 5
| row-size=33B cardinality=4
|
04:ANALYTIC
@@ -754,7 +754,7 @@ where rnk < 100 and int_col is not null
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 100
+| predicates: row_number() < 100, int_col IS NOT NULL
| row-size=31B cardinality=1.10K
|
02:ANALYTIC
@@ -819,7 +819,7 @@ where rnk <= 100 and int_col is not null
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() <= 100
+| predicates: int_col IS NOT NULL, row_number() <= 100
| row-size=31B cardinality=10
|
02:ANALYTIC
@@ -840,7 +840,7 @@ PLAN-ROOT SINK
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() <= 100
+| predicates: int_col IS NOT NULL, row_number() <= 100
| row-size=31B cardinality=10
|
02:ANALYTIC
@@ -872,7 +872,7 @@ where rnk = 42 and int_col is not null
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() = 42
+| predicates: int_col IS NOT NULL, row_number() = 42
| row-size=31B cardinality=4
|
02:ANALYTIC
@@ -893,7 +893,7 @@ PLAN-ROOT SINK
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() = 42
+| predicates: int_col IS NOT NULL, row_number() = 42
| row-size=31B cardinality=4
|
02:ANALYTIC
@@ -1061,7 +1061,7 @@ where rnk < 5 and int_col is not null
PLAN-ROOT SINK
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 5
+| predicates: int_col IS NOT NULL, row_number() < 5
| row-size=37B cardinality=1
|
02:ANALYTIC
@@ -1087,7 +1087,7 @@ PLAN-ROOT SINK
06:EXCHANGE [UNPARTITIONED]
|
03:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 5
+| predicates: int_col IS NOT NULL, row_number() < 5
| row-size=37B cardinality=1
|
02:ANALYTIC
@@ -1128,7 +1128,7 @@ where rnk < 5 and int_col is not null
PLAN-ROOT SINK
|
05:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 5
+| predicates: row_number() < 5, int_col IS NOT NULL
| row-size=52B cardinality=1.10K
|
04:ANALYTIC
@@ -1170,7 +1170,7 @@ where rnk < 5 and int_col is not null
PLAN-ROOT SINK
|
05:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 5
+| predicates: int_col IS NOT NULL, row_number() < 5
| row-size=37B cardinality=383
|
04:ANALYTIC
@@ -1207,7 +1207,7 @@ PLAN-ROOT SINK
09:EXCHANGE [UNPARTITIONED]
|
05:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 5
+| predicates: int_col IS NOT NULL, row_number() < 5
| row-size=37B cardinality=383
|
04:ANALYTIC
@@ -1262,7 +1262,7 @@ where rnk < 5 and int_col is not null
PLAN-ROOT SINK
|
04:SELECT
-| predicates: is_not_null_pred(int_col), row_number() < 5
+| predicates: row_number() < 5, int_col IS NOT NULL
| row-size=37B cardinality=1.10K
|
03:ANALYTIC
@@ -1657,11 +1657,11 @@ PLAN-ROOT SINK
05:HASH JOIN [INNER JOIN]
| hash predicates: functional.alltypesagg.id = id
| runtime filters: RF000 <- id
-| row-size=31B cardinality=106
+| row-size=31B cardinality=11
|
|--04:SELECT
-| | predicates: row_number() < 100
-| | row-size=27B cardinality=99
+| | predicates: id IS NOT NULL, row_number() < 100
+| | row-size=27B cardinality=10
| |
| 03:ANALYTIC
| | functions: row_number()
@@ -1680,6 +1680,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional.alltypesagg]
HDFS partitions=11/11 files=11 size=814.73KB
+ predicates: functional.alltypesagg.id IS NOT NULL
runtime filters: RF000 -> functional.alltypesagg.id
row-size=4B cardinality=11.00K
---- DISTRIBUTEDPLAN
@@ -1690,13 +1691,13 @@ PLAN-ROOT SINK
05:HASH JOIN [INNER JOIN, BROADCAST]
| hash predicates: functional.alltypesagg.id = id
| runtime filters: RF000 <- id
-| row-size=31B cardinality=106
+| row-size=31B cardinality=11
|
|--07:EXCHANGE [BROADCAST]
| |
| 04:SELECT
-| | predicates: row_number() < 100
-| | row-size=27B cardinality=99
+| | predicates: id IS NOT NULL, row_number() < 100
+| | row-size=27B cardinality=10
| |
| 03:ANALYTIC
| | functions: row_number()
@@ -1719,6 +1720,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional.alltypesagg]
HDFS partitions=11/11 files=11 size=814.73KB
+ predicates: functional.alltypesagg.id IS NOT NULL
runtime filters: RF000 -> functional.alltypesagg.id
row-size=4B cardinality=11.00K
====
@@ -1733,11 +1735,11 @@ PLAN-ROOT SINK
05:HASH JOIN [INNER JOIN]
| hash predicates: functional.alltypesagg.id = id
| runtime filters: RF000 <- id
-| row-size=31B cardinality=106
+| row-size=31B cardinality=11
|
|--04:SELECT
-| | predicates: row_number() < 100
-| | row-size=27B cardinality=99
+| | predicates: id IS NOT NULL, row_number() < 100
+| | row-size=27B cardinality=10
| |
| 03:ANALYTIC
| | functions: row_number()
@@ -1756,6 +1758,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional.alltypesagg]
HDFS partitions=11/11 files=11 size=814.73KB
+ predicates: functional.alltypesagg.id IS NOT NULL
runtime filters: RF000 -> functional.alltypesagg.id
row-size=4B cardinality=11.00K
---- DISTRIBUTEDPLAN
@@ -1766,13 +1769,13 @@ PLAN-ROOT SINK
05:HASH JOIN [INNER JOIN, BROADCAST]
| hash predicates: functional.alltypesagg.id = id
| runtime filters: RF000 <- id
-| row-size=31B cardinality=106
+| row-size=31B cardinality=11
|
|--07:EXCHANGE [BROADCAST]
| |
| 04:SELECT
-| | predicates: row_number() < 100
-| | row-size=27B cardinality=99
+| | predicates: id IS NOT NULL, row_number() < 100
+| | row-size=27B cardinality=10
| |
| 03:ANALYTIC
| | functions: row_number()
@@ -1795,6 +1798,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional.alltypesagg]
HDFS partitions=11/11 files=11 size=814.73KB
+ predicates: functional.alltypesagg.id IS NOT NULL
runtime filters: RF000 -> functional.alltypesagg.id
row-size=4B cardinality=11.00K
====
@@ -1817,14 +1821,14 @@ PLAN-ROOT SINK
|
|--10:AGGREGATE [FINALIZE]
| | group by: ID, RNK
-| | row-size=12B cardinality=198
+| | row-size=12B cardinality=20
| |
| 01:UNION
-| | row-size=12B cardinality=198
+| | row-size=12B cardinality=20
| |
| |--09:SELECT
-| | | predicates: row_number() < 100
-| | | row-size=27B cardinality=99
+| | | predicates: id IS NOT NULL, row_number() < 100
+| | | row-size=27B cardinality=10
| | |
| | 08:ANALYTIC
| | | functions: row_number()
@@ -1842,8 +1846,8 @@ PLAN-ROOT SINK
| | row-size=19B cardinality=11.00K
| |
| 05:SELECT
-| | predicates: row_number() < 100
-| | row-size=27B cardinality=99
+| | predicates: id IS NOT NULL, row_number() < 100
+| | row-size=27B cardinality=10
| |
| 04:ANALYTIC
| | functions: row_number()
@@ -1862,6 +1866,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional.alltypesagg]
HDFS partitions=11/11 files=11 size=814.73KB
+ predicates: functional.alltypesagg.id IS NOT NULL
runtime filters: RF000 -> functional.alltypesagg.id
row-size=95B cardinality=11.00K
---- DISTRIBUTEDPLAN
@@ -1878,14 +1883,14 @@ PLAN-ROOT SINK
| |
| 10:AGGREGATE [FINALIZE]
| | group by: ID, RNK
-| | row-size=12B cardinality=198
+| | row-size=12B cardinality=20
| |
| 01:UNION
-| | row-size=12B cardinality=198
+| | row-size=12B cardinality=20
| |
| |--09:SELECT
-| | | predicates: row_number() < 100
-| | | row-size=27B cardinality=99
+| | | predicates: id IS NOT NULL, row_number() < 100
+| | | row-size=27B cardinality=10
| | |
| | 08:ANALYTIC
| | | functions: row_number()
@@ -1907,8 +1912,8 @@ PLAN-ROOT SINK
| | row-size=19B cardinality=11.00K
| |
| 05:SELECT
-| | predicates: row_number() < 100
-| | row-size=27B cardinality=99
+| | predicates: id IS NOT NULL, row_number() < 100
+| | row-size=27B cardinality=10
| |
| 04:ANALYTIC
| | functions: row_number()
@@ -1931,6 +1936,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional.alltypesagg]
HDFS partitions=11/11 files=11 size=814.73KB
+ predicates: functional.alltypesagg.id IS NOT NULL
runtime filters: RF000 -> functional.alltypesagg.id
row-size=95B cardinality=11.00K
====
@@ -1947,11 +1953,11 @@ PLAN-ROOT SINK
07:HASH JOIN [INNER JOIN]
| hash predicates: functional.alltypesagg.id = id
| runtime filters: RF000 <- id
-| row-size=126B cardinality=106
+| row-size=126B cardinality=11
|
|--06:SELECT
-| | predicates: row_number() < 100
-| | row-size=31B cardinality=99
+| | predicates: id IS NOT NULL, row_number() < 100
+| | row-size=31B cardinality=10
| |
| 05:ANALYTIC
| | functions: row_number()
@@ -1978,6 +1984,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional.alltypesagg]
HDFS partitions=11/11 files=11 size=814.73KB
+ predicates: functional.alltypesagg.id IS NOT NULL
runtime filters: RF000 -> functional.alltypesagg.id
row-size=95B cardinality=11.00K
---- DISTRIBUTEDPLAN
@@ -1988,13 +1995,13 @@ PLAN-ROOT SINK
07:HASH JOIN [INNER JOIN, BROADCAST]
| hash predicates: functional.alltypesagg.id = id
| runtime filters: RF000 <- id
-| row-size=126B cardinality=106
+| row-size=126B cardinality=11
|
|--11:EXCHANGE [BROADCAST]
| |
| 06:SELECT
-| | predicates: row_number() < 100
-| | row-size=31B cardinality=99
+| | predicates: id IS NOT NULL, row_number() < 100
+| | row-size=31B cardinality=10
| |
| 05:ANALYTIC
| | functions: row_number()
@@ -2029,6 +2036,7 @@ PLAN-ROOT SINK
|
00:SCAN HDFS [functional.alltypesagg]
HDFS partitions=11/11 files=11 size=814.73KB
+ predicates: functional.alltypesagg.id IS NOT NULL
runtime filters: RF000 -> functional.alltypesagg.id
row-size=95B cardinality=11.00K
====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/calcite/limit-pushdown-analytic-calcite.test b/testdata/workloads/functional-planner/queries/PlannerTest/calcite/limit-pushdown-analytic-calcite.test
index 08f0066eb..4b0e36d67 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/calcite/limit-pushdown-analytic-calcite.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/calcite/limit-pushdown-analytic-calcite.test
@@ -722,7 +722,7 @@ PLAN-ROOT SINK
| row-size=22B cardinality=10
|
03:SELECT
-| predicates: rank() = 10 OR rank() = 20 OR rank() = 30
+| predicates: rank() IN (10, 20, 30)
| row-size=22B cardinality=11.00K
|
02:ANALYTIC
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q01.test b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q01.test
index 2eb4ce7bb..a31cd371d 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q01.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q01.test
@@ -25,11 +25,11 @@ limit 100
# end query 1 in stream 0 using template query1.tpl
---- PLAN
-Max Per-Host Resource Reservation: Memory=112.38MB Threads=1
-Per-Host Resource Estimates: Memory=1.18GB
+Max Per-Host Resource Reservation: Memory=111.91MB Threads=1
+Per-Host Resource Estimates: Memory=6.18GB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Instance Resources: mem-estimate=1.18GB mem-reservation=112.38MB thread-reservation=1 runtime-filters-memory=5.00MB
-| max-parallelism=1 segment-costs=[143540604, 143540604, 4705461, 12137693870, 100]
+| Per-Instance Resources: mem-estimate=6.18GB mem-reservation=111.91MB thread-reservation=1 runtime-filters-memory=5.00MB
+| max-parallelism=1 segment-costs=[977379059, 989786924, 43812422, 11059169531, 100]
PLAN-ROOT SINK
| output exprs: c_customer_id
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=100
@@ -37,7 +37,7 @@ PLAN-ROOT SINK
14:TOP-N [LIMIT=100]
| order by: c_customer_id ASC NULLS LAST
| mem-estimate=2.73KB mem-reservation=0B thread-reservation=0
-| tuple-ids=16 row-size=28B cardinality=100 cost=11991890128
+| tuple-ids=16 row-size=28B cardinality=100 cost=10924188257
| in pipelines: 14(GETNEXT), 00(OPEN)
|
13:HASH JOIN [INNER JOIN]
@@ -46,21 +46,21 @@ PLAN-ROOT SINK
| other join predicates: sum(tpcds_partitioned_parquet_snap.store_returns.sr_fee) > multiply(avg(sum(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee)), CAST(1.2 AS DECIMAL(2,1)))
| runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk, RF001[min_max] <- tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=0,6,7,15 row-size=94B cardinality=722.75M cost=131246886
+| tuple-ids=0,6,7,15 row-size=94B cardinality=661.28M cost=120084585
| in pipelines: 00(GETNEXT), 12(OPEN)
|
|--12:AGGREGATE [FINALIZE]
| | output: avg(sum(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee))
| | group by: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
| | mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=15 row-size=20B cardinality=653 cost=4705461
+| | tuple-ids=15 row-size=20B cardinality=653 cost=43812422
| | in pipelines: 12(GETNEXT), 11(OPEN)
| |
| 11:AGGREGATE [FINALIZE]
| | output: sum(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee)
| | group by: tpcds_partitioned_parquet_snap.store_returns_0.sr_customer_sk, tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
-| | mem-estimate=552.11MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=13 row-size=24B cardinality=16.08M cost=94921044
+| | mem-estimate=3.06GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=13 row-size=24B cardinality=149.78M cost=884092832
| | in pipelines: 11(GETNEXT), 08(OPEN)
| |
| 10:HASH JOIN [INNER JOIN]
@@ -68,42 +68,43 @@ PLAN-ROOT SINK
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_returns_0.sr_returned_date_sk = tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=9,11 row-size=24B cardinality=16.08M cost=7039125
+| | tuple-ids=9,11 row-size=24B cardinality=149.78M cost=65559183
| | in pipelines: 08(GETNEXT), 09(OPEN)
| |
| |--09:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=11 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=11 row-size=8B cardinality=373 cost=12520
| | in pipelines: 09(GETNEXT)
| |
| 08:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns]
-| HDFS partitions=2004/2004 files=2004 size=48.14GB
-| predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)
+| partition predicates: tpcds_partitioned_parquet_snap.store_returns_0.sr_returned_date_sk IS NOT NULL
+| HDFS partitions=2003/2004 files=2003 size=48.12GB
+| predicates: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk IS NOT NULL
| runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.store_returns_0.sr_returned_date_sk
| stored statistics:
| table: rows=863.99M size=48.14GB
-| partitions: 2004/2004 rows=863.99M
+| partitions: 2003/2003 rows=833.90M
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=30.09M est-scan-range=373(filtered from 2004)
-| parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)
-| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=9 row-size=16B cardinality=16.08M(filtered from 86.40M) cost=41569968
+| extrapolated-rows=disabled max-scan-range-rows=793.33K est-scan-range=373(filtered from 2003)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+| tuple-ids=9 row-size=16B cardinality=149.78M(filtered from 804.72M) cost=40122389
| in pipelines: 08(GETNEXT)
|
07:HASH JOIN [INNER JOIN]
| hash predicates: tpcds_partitioned_parquet_snap.customer.c_customer_sk = tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk
| fk/pk conjuncts: none
| runtime filters: RF002[bloom] <- tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk, RF003[min_max] <- tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk
-| mem-estimate=68.33MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=0,6,7 row-size=74B cardinality=1.11M cost=1591267
+| mem-estimate=64.56MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0,6,7 row-size=74B cardinality=1.01M cost=1455932
| in pipelines: 00(GETNEXT), 04(OPEN)
|
|--06:HASH JOIN [INNER JOIN]
@@ -111,28 +112,29 @@ PLAN-ROOT SINK
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_returns.sr_store_sk = tpcds_partitioned_parquet_snap.store.s_store_sk
| | runtime filters: RF004[bloom] <- tpcds_partitioned_parquet_snap.store.s_store_sk, RF005[min_max] <- tpcds_partitioned_parquet_snap.store.s_store_sk
| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=6,7 row-size=42B cardinality=1.11M cost=4325435
+| | tuple-ids=6,7 row-size=42B cardinality=1.01M cost=3957565
| | in pipelines: 04(GETNEXT), 05(OPEN)
| |
| |--05:SCAN HDFS [tpcds_partitioned_parquet_snap.store]
| | HDFS partitions=1/1 files=1 size=119.76KB
-| | predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN'
+| | predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN', tpcds_partitioned_parquet_snap.store.s_store_sk IS NOT NULL
| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store.s_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store.s_store_sk
| | stored statistics:
| | table: rows=1.35K size=119.76KB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=1.35K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN'
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN'
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN', tpcds_partitioned_parquet_snap.store.s_store_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
-| | tuple-ids=7 row-size=18B cardinality=46 cost=154
+| | tuple-ids=7 row-size=18B cardinality=46 cost=192
| | in pipelines: 05(GETNEXT)
| |
| 04:AGGREGATE [FINALIZE]
| | output: sum(tpcds_partitioned_parquet_snap.store_returns.sr_fee)
| | group by: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk, tpcds_partitioned_parquet_snap.store_returns.sr_store_sk
-| | mem-estimate=552.11MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=6 row-size=24B cardinality=16.08M cost=94921044
+| | having: sum(tpcds_partitioned_parquet_snap.store_returns.sr_fee) IS NOT NULL
+| | mem-estimate=3.01GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=6 row-size=24B cardinality=14.71M cost=868481164
| | in pipelines: 04(GETNEXT), 01(OPEN)
| |
| 03:HASH JOIN [INNER JOIN]
@@ -140,52 +142,55 @@ PLAN-ROOT SINK
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | runtime filters: RF006[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=2,4 row-size=24B cardinality=16.08M cost=7039125
+| | tuple-ids=2,4 row-size=24B cardinality=147.14M cost=64401519
| | in pipelines: 01(GETNEXT), 02(OPEN)
| |
| |--02:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=4 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=4 row-size=8B cardinality=373 cost=12520
| | in pipelines: 02(GETNEXT)
| |
| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns]
-| HDFS partitions=2004/2004 files=2004 size=48.14GB
-| predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)
+| partition predicates: tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk IS NOT NULL
+| HDFS partitions=2003/2004 files=2003 size=48.12GB
+| predicates: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_returns.sr_store_sk IS NOT NULL
| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF005[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF006[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk
| stored statistics:
| table: rows=863.99M size=48.14GB
-| partitions: 2004/2004 rows=863.99M
+| partitions: 2003/2003 rows=833.90M
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=30.09M est-scan-range=373(filtered from 2004)
-| parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)
-| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=2 row-size=16B cardinality=16.08M(filtered from 86.40M) cost=41569968
+| extrapolated-rows=disabled max-scan-range-rows=793.33K est-scan-range=373(filtered from 2003)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_returns.sr_store_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+| tuple-ids=2 row-size=16B cardinality=147.14M(filtered from 790.51M) cost=44483856
| in pipelines: 01(GETNEXT)
|
00:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
HDFS partitions=1/1 files=1 size=1.55GB
+ predicates: tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
runtime filters: RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.customer.c_customer_sk
stored statistics:
table: rows=30.00M size=1.55GB
columns: all
extrapolated-rows=disabled max-scan-range-rows=3.10M
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
- tuple-ids=0 row-size=32B cardinality=1.11M(filtered from 30.00M) cost=8640000
+ tuple-ids=0 row-size=32B cardinality=1.01M(filtered from 30.00M) cost=9483000
in pipelines: 00(GETNEXT)
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=272.58MB Threads=17
-Per-Host Resource Estimates: Memory=526MB
+Max Per-Host Resource Reservation: Memory=1.46GB Threads=51
+Per-Host Resource Estimates: Memory=2.84GB
F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=4.03MB mem-reservation=4.00MB thread-reservation=1
-| max-parallelism=1 segment-costs=[130] cpu-comparison-result=51 [max(1 (self) vs 51 (sum children))]
+| max-parallelism=1 segment-costs=[130] cpu-comparison-result=211 [max(1 (self) vs 211 (sum children))]
PLAN-ROOT SINK
| output exprs: c_customer_id
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=100
@@ -200,11 +205,11 @@ PLAN-ROOT SINK
F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
Per-Instance Resources: mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=1
-max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [max(10 (self) vs 51 (sum children))]
+max-parallelism=10 segment-costs=[11054198439, 104] cpu-comparison-result=211 [max(10 (self) vs 211 (sum children))]
14:TOP-N [LIMIT=100]
| order by: c_customer_id ASC NULLS LAST
| mem-estimate=2.73KB mem-reservation=0B thread-reservation=0
-| tuple-ids=16 row-size=28B cardinality=100 cost=11991890128
+| tuple-ids=16 row-size=28B cardinality=100 cost=10924188257
| in pipelines: 14(GETNEXT), 00(OPEN)
|
13:HASH JOIN [INNER JOIN, BROADCAST]
@@ -213,12 +218,12 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| fk/pk conjuncts: none
| other join predicates: sum(tpcds_partitioned_parquet_snap.store_returns.sr_fee) > multiply(avg(sum(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee)), CAST(1.2 AS DECIMAL(2,1)))
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=0,6,7,15 row-size=94B cardinality=722.75M cost=131246233
+| tuple-ids=0,6,7,15 row-size=94B cardinality=661.28M cost=120083932
| in pipelines: 00(GETNEXT), 24(OPEN)
|
|--F10:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | Per-Instance Resources: mem-estimate=3.10MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
-| | max-parallelism=10 segment-costs=[1513] cpu-comparison-result=20 [max(10 (self) vs 20 (sum children))]
+| | max-parallelism=10 segment-costs=[1513] cpu-comparison-result=100 [max(10 (self) vs 100 (sum children))]
| JOIN BUILD
| | join-table-id=00 plan-id=01 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
@@ -231,8 +236,8 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | in pipelines: 24(GETNEXT)
| |
| F08:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)] hosts=10 instances=10 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=10.38MB mem-reservation=1.94MB thread-reservation=1
-| max-parallelism=10 segment-costs=[40536, 49] cpu-comparison-result=20 [max(10 (self) vs 20 (sum children))]
+| Per-Instance Resources: mem-estimate=12.49MB mem-reservation=1.94MB thread-reservation=1
+| max-parallelism=10 segment-costs=[40536, 49] cpu-comparison-result=100 [max(10 (self) vs 100 (sum children))]
| 24:AGGREGATE [FINALIZE]
| | output: avg:merge(sum(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee))
| | group by: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
@@ -241,41 +246,41 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | in pipelines: 24(GETNEXT), 22(OPEN)
| |
| 23:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)]
-| | mem-estimate=393.05KB mem-reservation=0B thread-reservation=0
+| | mem-estimate=2.49MB mem-reservation=0B thread-reservation=0
| | tuple-ids=14 row-size=20B cardinality=78.36K cost=15914
| | in pipelines: 22(GETNEXT)
| |
-| F07:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_returns_0.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)] hosts=10 instances=10 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=75.76MB mem-reservation=36.00MB thread-reservation=1
-| max-parallelism=10 segment-costs=[98483035, 4908058, 141298] cpu-comparison-result=20 [max(10 (self) vs 20 (sum children))]
+| F07:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_returns_0.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)] hosts=10 instances=100 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=74.16MB mem-reservation=36.00MB thread-reservation=1
+| max-parallelism=100 segment-costs=[917269154, 44015019, 141298] cpu-comparison-result=100 [max(100 (self) vs 100 (sum children))]
| 12:AGGREGATE [STREAMING]
| | output: avg(sum(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee))
| | group by: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
| | mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=14 row-size=20B cardinality=78.36K cost=4908058
+| | tuple-ids=14 row-size=20B cardinality=78.36K cost=44015019
| | in pipelines: 22(GETNEXT)
| |
| 22:AGGREGATE [FINALIZE]
| | output: sum:merge(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee)
| | group by: tpcds_partitioned_parquet_snap.store_returns_0.sr_customer_sk, tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
-| | mem-estimate=55.21MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=13 row-size=24B cardinality=16.08M cost=94921044
+| | mem-estimate=51.42MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=13 row-size=24B cardinality=149.78M cost=884092832
| | in pipelines: 22(GETNEXT), 08(OPEN)
| |
| 21:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_returns_0.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)]
-| | mem-estimate=10.55MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=13 row-size=24B cardinality=16.08M cost=3561991
+| | mem-estimate=12.73MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=13 row-size=24B cardinality=149.78M cost=33176322
| | in pipelines: 08(GETNEXT)
| |
-| F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=20 (adjusted from 120)
+| F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=100 (adjusted from 120)
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-| Per-Instance Resources: mem-estimate=51.09MB mem-reservation=34.50MB thread-reservation=1
-| max-parallelism=20 segment-costs=[143529764, 33140190] cpu-comparison-result=20 [max(20 (self) vs 11 (sum children))]
+| Per-Instance Resources: mem-estimate=65.50MB mem-reservation=34.03MB thread-reservation=1
+| max-parallelism=100 segment-costs=[989774031, 308667114] cpu-comparison-result=100 [max(100 (self) vs 11 (sum children))]
| 11:AGGREGATE [STREAMING]
| | output: sum(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee)
| | group by: tpcds_partitioned_parquet_snap.store_returns_0.sr_customer_sk, tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
-| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=13 row-size=24B cardinality=16.08M cost=94921044
+| | mem-estimate=38.57MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=13 row-size=24B cardinality=149.78M cost=884092832
| | in pipelines: 08(GETNEXT)
| |
| 10:HASH JOIN [INNER JOIN, BROADCAST]
@@ -283,17 +288,17 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | hash predicates: tpcds_partitioned_parquet_snap.store_returns_0.sr_returned_date_sk = tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_returns_0.sr_returned_date_sk = tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=9,11 row-size=24B cardinality=16.08M cost=7038752
+| | tuple-ids=9,11 row-size=24B cardinality=149.78M cost=65558810
| | in pipelines: 08(GETNEXT), 09(OPEN)
| |
| |--F11:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=4.89MB mem-reservation=4.88MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | Per-Instance Resources: mem-estimate=20.39MB mem-reservation=20.38MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
| | | join-table-id=01 plan-id=02 cohort-id=02
| | | build expressions: tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
-| | | mem-estimate=3.88MB mem-reservation=3.88MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | | mem-estimate=19.38MB mem-reservation=19.38MB spill-buffer=64.00KB thread-reservation=0 cost=373
| | |
| | 20:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
@@ -302,32 +307,33 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | |
| | F06:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
+| | max-parallelism=1 segment-costs=[12536]
| | 09:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=11 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=11 row-size=8B cardinality=373 cost=12520
| | in pipelines: 09(GETNEXT)
| |
| 08:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns, RANDOM]
-| HDFS partitions=2004/2004 files=2004 size=48.14GB
-| predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)
+| partition predicates: tpcds_partitioned_parquet_snap.store_returns_0.sr_returned_date_sk IS NOT NULL
+| HDFS partitions=2003/2004 files=2003 size=48.12GB
+| predicates: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk IS NOT NULL
| runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.store_returns_0.sr_returned_date_sk
| stored statistics:
| table: rows=863.99M size=48.14GB
-| partitions: 2004/2004 rows=863.99M
+| partitions: 2003/2003 rows=833.90M
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=30.09M est-scan-range=373(filtered from 2004)
-| parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)
-| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=9 row-size=16B cardinality=16.08M(filtered from 86.40M) cost=41569968
+| extrapolated-rows=disabled max-scan-range-rows=793.33K est-scan-range=373(filtered from 2003)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+| tuple-ids=9 row-size=16B cardinality=149.78M(filtered from 804.72M) cost=40122389
| in pipelines: 08(GETNEXT)
|
07:HASH JOIN [INNER JOIN, BROADCAST]
@@ -335,42 +341,42 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| hash predicates: tpcds_partitioned_parquet_snap.customer.c_customer_sk = tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk
| fk/pk conjuncts: none
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=0,6,7 row-size=74B cardinality=1.11M cost=484452
+| tuple-ids=0,6,7 row-size=74B cardinality=1.01M cost=443250
| in pipelines: 00(GETNEXT), 17(OPEN)
|
|--F12:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Instance Resources: mem-estimate=79.82MB mem-reservation=35.00MB thread-reservation=1 runtime-filters-memory=1.00MB
-| | max-parallelism=10 segment-costs=[2577765] cpu-comparison-result=31 [max(10 (self) vs 31 (sum children))]
+| | Per-Instance Resources: mem-estimate=80.45MB mem-reservation=35.00MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[2358532] cpu-comparison-result=111 [max(10 (self) vs 111 (sum children))]
| JOIN BUILD
| | join-table-id=02 plan-id=03 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk
| | runtime filters: RF002[bloom] <- tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk, RF003[min_max] <- tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk
-| | mem-estimate=68.33MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=1106815
+| | mem-estimate=64.56MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=1012682
| |
| 19:EXCHANGE [BROADCAST]
-| | mem-estimate=10.49MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=6,7 row-size=42B cardinality=1.11M cost=1470950
+| | mem-estimate=14.88MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=6,7 row-size=42B cardinality=1.01M cost=1345850
| | in pipelines: 17(GETNEXT)
| |
-| F03:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)] hosts=10 instances=10 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=65.76MB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=10 segment-costs=[98483035, 4498716] cpu-comparison-result=31 [max(10 (self) vs 31 (sum children))]
+| F03:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)] hosts=10 instances=100 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=63.25MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=100 segment-costs=[901071645, 4116105] cpu-comparison-result=111 [max(100 (self) vs 111 (sum children))]
| 06:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=03
| | hash predicates: tpcds_partitioned_parquet_snap.store_returns.sr_store_sk = tpcds_partitioned_parquet_snap.store.s_store_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_returns.sr_store_sk = tpcds_partitioned_parquet_snap.store.s_store_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=6,7 row-size=42B cardinality=1.11M cost=4325389
+| | tuple-ids=6,7 row-size=42B cardinality=1.01M cost=3957519
| | in pipelines: 17(GETNEXT), 05(OPEN)
| |
| |--F13:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=2.95MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | Per-Instance Resources: mem-estimate=20.39MB mem-reservation=20.38MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[106]
| | JOIN BUILD
| | | join-table-id=03 plan-id=04 cohort-id=03
| | | build expressions: tpcds_partitioned_parquet_snap.store.s_store_sk
| | | runtime filters: RF004[bloom] <- tpcds_partitioned_parquet_snap.store.s_store_sk, RF005[min_max] <- tpcds_partitioned_parquet_snap.store.s_store_sk
-| | | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0 cost=46
+| | | mem-estimate=19.38MB mem-reservation=19.38MB spill-buffer=64.00KB thread-reservation=0 cost=46
| | |
| | 18:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
@@ -380,42 +386,43 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | F04:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
| | Per-Instance Resources: mem-estimate=16.09MB mem-reservation=16.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[157]
+| | max-parallelism=1 segment-costs=[195]
| | 05:SCAN HDFS [tpcds_partitioned_parquet_snap.store, RANDOM]
| | HDFS partitions=1/1 files=1 size=119.76KB
-| | predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN'
+| | predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN', tpcds_partitioned_parquet_snap.store.s_store_sk IS NOT NULL
| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store.s_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store.s_store_sk
| | stored statistics:
| | table: rows=1.35K size=119.76KB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=1.35K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN'
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN'
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN', tpcds_partitioned_parquet_snap.store.s_store_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
-| | tuple-ids=7 row-size=18B cardinality=46 cost=154
+| | tuple-ids=7 row-size=18B cardinality=46 cost=192
| | in pipelines: 05(GETNEXT)
| |
| 17:AGGREGATE [FINALIZE]
| | output: sum:merge(tpcds_partitioned_parquet_snap.store_returns.sr_fee)
| | group by: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk, tpcds_partitioned_parquet_snap.store_returns.sr_store_sk
-| | mem-estimate=55.21MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=6 row-size=24B cardinality=16.08M cost=94921044
+| | having: sum(tpcds_partitioned_parquet_snap.store_returns.sr_fee) IS NOT NULL
+| | mem-estimate=50.51MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=6 row-size=24B cardinality=14.71M cost=868481164
| | in pipelines: 17(GETNEXT), 01(OPEN)
| |
| 16:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)]
-| | mem-estimate=10.55MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=6 row-size=24B cardinality=16.08M cost=3561991
+| | mem-estimate=12.73MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=6 row-size=24B cardinality=147.14M cost=32590481
| | in pipelines: 01(GETNEXT)
| |
-| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=20 (adjusted from 120)
+| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=100 (adjusted from 120)
| Per-Host Shared Resources: mem-estimate=3.00MB mem-reservation=3.00MB thread-reservation=0 runtime-filters-memory=3.00MB
-| Per-Instance Resources: mem-estimate=51.09MB mem-reservation=34.50MB thread-reservation=1
-| max-parallelism=20 segment-costs=[143529764, 33140190] cpu-comparison-result=20 [max(20 (self) vs 11 (sum children))]
+| Per-Instance Resources: mem-estimate=64.82MB mem-reservation=34.03MB thread-reservation=1
+| max-parallelism=100 segment-costs=[977366166, 303216545] cpu-comparison-result=100 [max(100 (self) vs 11 (sum children))]
| 04:AGGREGATE [STREAMING]
| | output: sum(tpcds_partitioned_parquet_snap.store_returns.sr_fee)
| | group by: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk, tpcds_partitioned_parquet_snap.store_returns.sr_store_sk
-| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=6 row-size=24B cardinality=16.08M cost=94921044
+| | mem-estimate=37.89MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=6 row-size=24B cardinality=147.14M cost=868481164
| | in pipelines: 01(GETNEXT)
| |
| 03:HASH JOIN [INNER JOIN, BROADCAST]
@@ -423,17 +430,17 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | hash predicates: tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=2,4 row-size=24B cardinality=16.08M cost=7038752
+| | tuple-ids=2,4 row-size=24B cardinality=147.14M cost=64401146
| | in pipelines: 01(GETNEXT), 02(OPEN)
| |
| |--F14:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=4.89MB mem-reservation=4.88MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | Per-Instance Resources: mem-estimate=20.39MB mem-reservation=20.38MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
| | | join-table-id=04 plan-id=05 cohort-id=03
| | | build expressions: tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | | runtime filters: RF006[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
-| | | mem-estimate=3.88MB mem-reservation=3.88MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | | mem-estimate=19.38MB mem-reservation=19.38MB spill-buffer=64.00KB thread-reservation=0 cost=373
| | |
| | 15:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
@@ -442,50 +449,53 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | |
| | F02:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
+| | max-parallelism=1 segment-costs=[12536]
| | 02:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=4 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=4 row-size=8B cardinality=373 cost=12520
| | in pipelines: 02(GETNEXT)
| |
| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns, RANDOM]
-| HDFS partitions=2004/2004 files=2004 size=48.14GB
-| predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)
+| partition predicates: tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk IS NOT NULL
+| HDFS partitions=2003/2004 files=2003 size=48.12GB
+| predicates: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_returns.sr_store_sk IS NOT NULL
| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF005[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF006[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk
| stored statistics:
| table: rows=863.99M size=48.14GB
-| partitions: 2004/2004 rows=863.99M
+| partitions: 2003/2003 rows=833.90M
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=30.09M est-scan-range=373(filtered from 2004)
-| parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)
-| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=2 row-size=16B cardinality=16.08M(filtered from 86.40M) cost=41569968
+| extrapolated-rows=disabled max-scan-range-rows=793.33K est-scan-range=373(filtered from 2003)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_returns.sr_store_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+| tuple-ids=2 row-size=16B cardinality=147.14M(filtered from 790.51M) cost=44483856
| in pipelines: 01(GETNEXT)
|
00:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
HDFS partitions=1/1 files=1 size=1.55GB
+ predicates: tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
runtime filters: RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.customer.c_customer_sk
stored statistics:
table: rows=30.00M size=1.55GB
columns: all
extrapolated-rows=disabled max-scan-range-rows=3.10M
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
- tuple-ids=0 row-size=32B cardinality=1.11M(filtered from 30.00M) cost=8640000
+ tuple-ids=0 row-size=32B cardinality=1.01M(filtered from 30.00M) cost=9483000
in pipelines: 00(GETNEXT)
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=272.58MB Threads=17
-Per-Host Resource Estimates: Memory=526MB
+Max Per-Host Resource Reservation: Memory=1.46GB Threads=51
+Per-Host Resource Estimates: Memory=2.84GB
F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=4.03MB mem-reservation=4.00MB thread-reservation=1
-| max-parallelism=1 segment-costs=[130] cpu-comparison-result=51 [max(1 (self) vs 51 (sum children))]
+| max-parallelism=1 segment-costs=[130] cpu-comparison-result=211 [max(1 (self) vs 211 (sum children))]
PLAN-ROOT SINK
| output exprs: c_customer_id
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=100
@@ -500,11 +510,11 @@ PLAN-ROOT SINK
F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
Per-Instance Resources: mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=1
-max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [max(10 (self) vs 51 (sum children))]
+max-parallelism=10 segment-costs=[11054198439, 104] cpu-comparison-result=211 [max(10 (self) vs 211 (sum children))]
14:TOP-N [LIMIT=100]
| order by: c_customer_id ASC NULLS LAST
| mem-estimate=2.73KB mem-reservation=0B thread-reservation=0
-| tuple-ids=16 row-size=28B cardinality=100 cost=11991890128
+| tuple-ids=16 row-size=28B cardinality=100 cost=10924188257
| in pipelines: 14(GETNEXT), 00(OPEN)
|
13:HASH JOIN [INNER JOIN, BROADCAST]
@@ -513,12 +523,12 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| fk/pk conjuncts: none
| other join predicates: sum(tpcds_partitioned_parquet_snap.store_returns.sr_fee) > multiply(avg(sum(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee)), CAST(1.2 AS DECIMAL(2,1)))
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=0,6,7,15 row-size=94B cardinality=722.75M cost=131246233
+| tuple-ids=0,6,7,15 row-size=94B cardinality=661.28M cost=120083932
| in pipelines: 00(GETNEXT), 24(OPEN)
|
|--F10:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | Per-Instance Resources: mem-estimate=3.10MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
-| | max-parallelism=10 segment-costs=[1513] cpu-comparison-result=20 [max(10 (self) vs 20 (sum children))]
+| | max-parallelism=10 segment-costs=[1513] cpu-comparison-result=100 [max(10 (self) vs 100 (sum children))]
| JOIN BUILD
| | join-table-id=00 plan-id=01 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
@@ -531,8 +541,8 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | in pipelines: 24(GETNEXT)
| |
| F08:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)] hosts=10 instances=10 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=10.38MB mem-reservation=1.94MB thread-reservation=1
-| max-parallelism=10 segment-costs=[40536, 49] cpu-comparison-result=20 [max(10 (self) vs 20 (sum children))]
+| Per-Instance Resources: mem-estimate=12.49MB mem-reservation=1.94MB thread-reservation=1
+| max-parallelism=10 segment-costs=[40536, 49] cpu-comparison-result=100 [max(10 (self) vs 100 (sum children))]
| 24:AGGREGATE [FINALIZE]
| | output: avg:merge(sum(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee))
| | group by: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
@@ -541,41 +551,41 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | in pipelines: 24(GETNEXT), 22(OPEN)
| |
| 23:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)]
-| | mem-estimate=393.05KB mem-reservation=0B thread-reservation=0
+| | mem-estimate=2.49MB mem-reservation=0B thread-reservation=0
| | tuple-ids=14 row-size=20B cardinality=78.36K cost=15914
| | in pipelines: 22(GETNEXT)
| |
-| F07:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_returns_0.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)] hosts=10 instances=10 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=75.76MB mem-reservation=36.00MB thread-reservation=1
-| max-parallelism=10 segment-costs=[98483035, 4908058, 141298] cpu-comparison-result=20 [max(10 (self) vs 20 (sum children))]
+| F07:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_returns_0.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)] hosts=10 instances=100 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=74.16MB mem-reservation=36.00MB thread-reservation=1
+| max-parallelism=100 segment-costs=[917269154, 44015019, 141298] cpu-comparison-result=100 [max(100 (self) vs 100 (sum children))]
| 12:AGGREGATE [STREAMING]
| | output: avg(sum(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee))
| | group by: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
| | mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=14 row-size=20B cardinality=78.36K cost=4908058
+| | tuple-ids=14 row-size=20B cardinality=78.36K cost=44015019
| | in pipelines: 22(GETNEXT)
| |
| 22:AGGREGATE [FINALIZE]
| | output: sum:merge(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee)
| | group by: tpcds_partitioned_parquet_snap.store_returns_0.sr_customer_sk, tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
-| | mem-estimate=55.21MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=13 row-size=24B cardinality=16.08M cost=94921044
+| | mem-estimate=51.42MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=13 row-size=24B cardinality=149.78M cost=884092832
| | in pipelines: 22(GETNEXT), 08(OPEN)
| |
| 21:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_returns_0.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)]
-| | mem-estimate=10.55MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=13 row-size=24B cardinality=16.08M cost=3561991
+| | mem-estimate=12.73MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=13 row-size=24B cardinality=149.78M cost=33176322
| | in pipelines: 08(GETNEXT)
| |
-| F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=20 (adjusted from 120)
+| F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=100 (adjusted from 120)
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-| Per-Instance Resources: mem-estimate=51.09MB mem-reservation=34.50MB thread-reservation=1
-| max-parallelism=20 segment-costs=[143529764, 33140190] cpu-comparison-result=20 [max(20 (self) vs 11 (sum children))]
+| Per-Instance Resources: mem-estimate=65.50MB mem-reservation=34.03MB thread-reservation=1
+| max-parallelism=100 segment-costs=[989774031, 308667114] cpu-comparison-result=100 [max(100 (self) vs 11 (sum children))]
| 11:AGGREGATE [STREAMING]
| | output: sum(tpcds_partitioned_parquet_snap.store_returns_0.sr_fee)
| | group by: tpcds_partitioned_parquet_snap.store_returns_0.sr_customer_sk, tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk
-| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=13 row-size=24B cardinality=16.08M cost=94921044
+| | mem-estimate=38.57MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=13 row-size=24B cardinality=149.78M cost=884092832
| | in pipelines: 08(GETNEXT)
| |
| 10:HASH JOIN [INNER JOIN, BROADCAST]
@@ -583,17 +593,17 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | hash predicates: tpcds_partitioned_parquet_snap.store_returns_0.sr_returned_date_sk = tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_returns_0.sr_returned_date_sk = tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=9,11 row-size=24B cardinality=16.08M cost=7038752
+| | tuple-ids=9,11 row-size=24B cardinality=149.78M cost=65558810
| | in pipelines: 08(GETNEXT), 09(OPEN)
| |
| |--F11:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=4.89MB mem-reservation=4.88MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | Per-Instance Resources: mem-estimate=20.39MB mem-reservation=20.38MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
| | | join-table-id=01 plan-id=02 cohort-id=02
| | | build expressions: tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
-| | | mem-estimate=3.88MB mem-reservation=3.88MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | | mem-estimate=19.38MB mem-reservation=19.38MB spill-buffer=64.00KB thread-reservation=0 cost=373
| | |
| | 20:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
@@ -602,32 +612,33 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | |
| | F06:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
+| | max-parallelism=1 segment-costs=[12536]
| | 09:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = CAST(2000 AS INT), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=11 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=11 row-size=8B cardinality=373 cost=12520
| | in pipelines: 09(GETNEXT)
| |
| 08:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns, RANDOM]
-| HDFS partitions=2004/2004 files=2004 size=48.14GB
-| predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)
+| partition predicates: tpcds_partitioned_parquet_snap.store_returns_0.sr_returned_date_sk IS NOT NULL
+| HDFS partitions=2003/2004 files=2003 size=48.12GB
+| predicates: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk IS NOT NULL
| runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.store_returns_0.sr_returned_date_sk
| stored statistics:
| table: rows=863.99M size=48.14GB
-| partitions: 2004/2004 rows=863.99M
+| partitions: 2003/2003 rows=833.90M
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=30.09M est-scan-range=373(filtered from 2004)
-| parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk)
-| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=9 row-size=16B cardinality=16.08M(filtered from 86.40M) cost=41569968
+| extrapolated-rows=disabled max-scan-range-rows=793.33K est-scan-range=373(filtered from 2003)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_returns_0.sr_store_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+| tuple-ids=9 row-size=16B cardinality=149.78M(filtered from 804.72M) cost=40122389
| in pipelines: 08(GETNEXT)
|
07:HASH JOIN [INNER JOIN, BROADCAST]
@@ -635,42 +646,42 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| hash predicates: tpcds_partitioned_parquet_snap.customer.c_customer_sk = tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk
| fk/pk conjuncts: none
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=0,6,7 row-size=74B cardinality=1.11M cost=484452
+| tuple-ids=0,6,7 row-size=74B cardinality=1.01M cost=443250
| in pipelines: 00(GETNEXT), 17(OPEN)
|
|--F12:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Instance Resources: mem-estimate=79.82MB mem-reservation=35.00MB thread-reservation=1 runtime-filters-memory=1.00MB
-| | max-parallelism=10 segment-costs=[2577765] cpu-comparison-result=31 [max(10 (self) vs 31 (sum children))]
+| | Per-Instance Resources: mem-estimate=80.45MB mem-reservation=35.00MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | max-parallelism=10 segment-costs=[2358532] cpu-comparison-result=111 [max(10 (self) vs 111 (sum children))]
| JOIN BUILD
| | join-table-id=02 plan-id=03 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk
| | runtime filters: RF002[bloom] <- tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk, RF003[min_max] <- tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk
-| | mem-estimate=68.33MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=1106815
+| | mem-estimate=64.56MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=1012682
| |
| 19:EXCHANGE [BROADCAST]
-| | mem-estimate=10.49MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=6,7 row-size=42B cardinality=1.11M cost=1470950
+| | mem-estimate=14.88MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=6,7 row-size=42B cardinality=1.01M cost=1345850
| | in pipelines: 17(GETNEXT)
| |
-| F03:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)] hosts=10 instances=10 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=65.76MB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=10 segment-costs=[98483035, 4498716] cpu-comparison-result=31 [max(10 (self) vs 31 (sum children))]
+| F03:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)] hosts=10 instances=100 (adjusted from 120)
+| Per-Instance Resources: mem-estimate=63.25MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=100 segment-costs=[901071645, 4116105] cpu-comparison-result=111 [max(100 (self) vs 111 (sum children))]
| 06:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=03
| | hash predicates: tpcds_partitioned_parquet_snap.store_returns.sr_store_sk = tpcds_partitioned_parquet_snap.store.s_store_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_returns.sr_store_sk = tpcds_partitioned_parquet_snap.store.s_store_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=6,7 row-size=42B cardinality=1.11M cost=4325389
+| | tuple-ids=6,7 row-size=42B cardinality=1.01M cost=3957519
| | in pipelines: 17(GETNEXT), 05(OPEN)
| |
| |--F13:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=2.95MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | Per-Instance Resources: mem-estimate=20.39MB mem-reservation=20.38MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[106]
| | JOIN BUILD
| | | join-table-id=03 plan-id=04 cohort-id=03
| | | build expressions: tpcds_partitioned_parquet_snap.store.s_store_sk
| | | runtime filters: RF004[bloom] <- tpcds_partitioned_parquet_snap.store.s_store_sk, RF005[min_max] <- tpcds_partitioned_parquet_snap.store.s_store_sk
-| | | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0 cost=46
+| | | mem-estimate=19.38MB mem-reservation=19.38MB spill-buffer=64.00KB thread-reservation=0 cost=46
| | |
| | 18:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
@@ -680,42 +691,43 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | F04:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
| | Per-Instance Resources: mem-estimate=16.09MB mem-reservation=16.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[157]
+| | max-parallelism=1 segment-costs=[195]
| | 05:SCAN HDFS [tpcds_partitioned_parquet_snap.store, RANDOM]
| | HDFS partitions=1/1 files=1 size=119.76KB
-| | predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN'
+| | predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN', tpcds_partitioned_parquet_snap.store.s_store_sk IS NOT NULL
| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store.s_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store.s_store_sk
| | stored statistics:
| | table: rows=1.35K size=119.76KB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=1.35K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN'
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN'
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.store.s_state = 'TN', tpcds_partitioned_parquet_snap.store.s_store_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
-| | tuple-ids=7 row-size=18B cardinality=46 cost=154
+| | tuple-ids=7 row-size=18B cardinality=46 cost=192
| | in pipelines: 05(GETNEXT)
| |
| 17:AGGREGATE [FINALIZE]
| | output: sum:merge(tpcds_partitioned_parquet_snap.store_returns.sr_fee)
| | group by: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk, tpcds_partitioned_parquet_snap.store_returns.sr_store_sk
-| | mem-estimate=55.21MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=6 row-size=24B cardinality=16.08M cost=94921044
+| | having: sum(tpcds_partitioned_parquet_snap.store_returns.sr_fee) IS NOT NULL
+| | mem-estimate=50.51MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=6 row-size=24B cardinality=14.71M cost=868481164
| | in pipelines: 17(GETNEXT), 01(OPEN)
| |
| 16:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk,tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)]
-| | mem-estimate=10.55MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=6 row-size=24B cardinality=16.08M cost=3561991
+| | mem-estimate=12.73MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=6 row-size=24B cardinality=147.14M cost=32590481
| | in pipelines: 01(GETNEXT)
| |
-| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=20 (adjusted from 120)
+| F01:PLAN FRAGMENT [RANDOM] hosts=10 instances=100 (adjusted from 120)
| Per-Host Shared Resources: mem-estimate=3.00MB mem-reservation=3.00MB thread-reservation=0 runtime-filters-memory=3.00MB
-| Per-Instance Resources: mem-estimate=51.09MB mem-reservation=34.50MB thread-reservation=1
-| max-parallelism=20 segment-costs=[143529764, 33140190] cpu-comparison-result=20 [max(20 (self) vs 11 (sum children))]
+| Per-Instance Resources: mem-estimate=64.82MB mem-reservation=34.03MB thread-reservation=1
+| max-parallelism=100 segment-costs=[977366166, 303216545] cpu-comparison-result=100 [max(100 (self) vs 11 (sum children))]
| 04:AGGREGATE [STREAMING]
| | output: sum(tpcds_partitioned_parquet_snap.store_returns.sr_fee)
| | group by: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk, tpcds_partitioned_parquet_snap.store_returns.sr_store_sk
-| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=6 row-size=24B cardinality=16.08M cost=94921044
+| | mem-estimate=37.89MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=6 row-size=24B cardinality=147.14M cost=868481164
| | in pipelines: 01(GETNEXT)
| |
| 03:HASH JOIN [INNER JOIN, BROADCAST]
@@ -723,17 +735,17 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | hash predicates: tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=2,4 row-size=24B cardinality=16.08M cost=7038752
+| | tuple-ids=2,4 row-size=24B cardinality=147.14M cost=64401146
| | in pipelines: 01(GETNEXT), 02(OPEN)
| |
| |--F14:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=4.89MB mem-reservation=4.88MB thread-reservation=1 runtime-filters-memory=1.00MB
+| | | Per-Instance Resources: mem-estimate=20.39MB mem-reservation=20.38MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
| | | join-table-id=04 plan-id=05 cohort-id=03
| | | build expressions: tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | | runtime filters: RF006[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
-| | | mem-estimate=3.88MB mem-reservation=3.88MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | | mem-estimate=19.38MB mem-reservation=19.38MB spill-buffer=64.00KB thread-reservation=0 cost=373
| | |
| | 15:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
@@ -742,42 +754,45 @@ max-parallelism=10 segment-costs=[12132260813, 104] cpu-comparison-result=51 [ma
| | |
| | F02:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
+| | max-parallelism=1 segment-costs=[12536]
| | 02:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2000 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=4 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=4 row-size=8B cardinality=373 cost=12520
| | in pipelines: 02(GETNEXT)
| |
| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns, RANDOM]
-| HDFS partitions=2004/2004 files=2004 size=48.14GB
-| predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)
+| partition predicates: tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk IS NOT NULL
+| HDFS partitions=2003/2004 files=2003 size=48.12GB
+| predicates: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_returns.sr_store_sk IS NOT NULL
| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF005[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF006[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk
| stored statistics:
| table: rows=863.99M size=48.14GB
-| partitions: 2004/2004 rows=863.99M
+| partitions: 2003/2003 rows=833.90M
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=30.09M est-scan-range=373(filtered from 2004)
-| parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.store_returns.sr_store_sk)
-| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=2 row-size=16B cardinality=16.08M(filtered from 86.40M) cost=41569968
+| extrapolated-rows=disabled max-scan-range-rows=793.33K est-scan-range=373(filtered from 2003)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_returns.sr_customer_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_returns.sr_store_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+| tuple-ids=2 row-size=16B cardinality=147.14M(filtered from 790.51M) cost=44483856
| in pipelines: 01(GETNEXT)
|
00:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
HDFS partitions=1/1 files=1 size=1.55GB
+ predicates: tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
runtime filters: RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.customer.c_customer_sk
stored statistics:
table: rows=30.00M size=1.55GB
columns: all
extrapolated-rows=disabled max-scan-range-rows=3.10M
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
- tuple-ids=0 row-size=32B cardinality=1.11M(filtered from 30.00M) cost=8640000
+ tuple-ids=0 row-size=32B cardinality=1.01M(filtered from 30.00M) cost=9483000
in pipelines: 00(GETNEXT)
====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q02.test b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q02.test
index 88a3cd3e7..ae28760d8 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q02.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q02.test
@@ -60,11 +60,11 @@ with wscs as
# end query 2 in stream 0 using template query2.tpl
---- PLAN
-Max Per-Host Resource Reservation: Memory=26.62MB Threads=1
+Max Per-Host Resource Reservation: Memory=25.12MB Threads=1
Per-Host Resource Estimates: Memory=114MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Instance Resources: mem-estimate=114.38MB mem-reservation=26.62MB thread-reservation=1 runtime-filters-memory=5.00MB
-| max-parallelism=1 segment-costs=[8530838007, 8530838007, 58178, 20480]
+| Per-Instance Resources: mem-estimate=114.38MB mem-reservation=25.12MB thread-reservation=1 runtime-filters-memory=5.00MB
+| max-parallelism=1 segment-costs=[8501768437, 8501768437, 62284, 20480]
PLAN-ROOT SINK
| output exprs: tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq, round(divide(sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Sunday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Sunday' THEN SALES_PRICE ELSE NULL END)), CAST(2 AS TINYINT)), round(divide(sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Monday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_sn [...]
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=20480
@@ -93,22 +93,22 @@ PLAN-ROOT SINK
| |
| |--14:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT), tpcds_partitioned_parquet_snap.date_dim_2.d_week_seq IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT), tpcds_partitioned_parquet_snap.date_dim_2.d_week_seq IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
-| | tuple-ids=18 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=18 row-size=8B cardinality=373 cost=12520
| | in pipelines: 14(GETNEXT)
| |
| 13:AGGREGATE [FINALIZE]
| | output: sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Sunday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Monday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Tuesday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Wednesday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitione [...]
| | group by: tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq
| | mem-estimate=10.00MB mem-reservation=2.88MB spill-buffer=128.00KB thread-reservation=0
-| | tuple-ids=17 row-size=116B cardinality=10.64K cost=4617060482
+| | tuple-ids=17 row-size=116B cardinality=10.64K cost=4601325105
| | in pipelines: 13(GETNEXT), 09(OPEN), 10(OPEN)
| |
| 12:HASH JOIN [INNER JOIN]
@@ -116,47 +116,51 @@ PLAN-ROOT SINK
| | fk/pk conjuncts: assumed fk/pk
| | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk
| | mem-estimate=4.75MB mem-reservation=4.75MB spill-buffer=256.00KB thread-reservation=0
-| | tuple-ids=14,15 row-size=35B cardinality=6.48G cost=2836389159
+| | tuple-ids=14,15 row-size=35B cardinality=6.46G cost=2826722669
| | in pipelines: 09(GETNEXT), 10(GETNEXT), 11(OPEN)
| |
| |--11:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| | HDFS partitions=1/1 files=1 size=2.17MB
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq IS NOT NULL
| | runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq, RF006[bloom] -> tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=15 row-size=27B cardinality=73.05K cost=15928
+| | tuple-ids=15 row-size=27B cardinality=73.05K cost=20034
| | in pipelines: 11(GETNEXT)
| |
| 08:UNION
| | mem-estimate=0B mem-reservation=0B thread-reservation=0
-| | tuple-ids=14 row-size=8B cardinality=6.48G cost=330871146
+| | tuple-ids=14 row-size=8B cardinality=6.46G cost=329743499
| | in pipelines: 09(GETNEXT), 10(GETNEXT)
| |
| |--10:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales]
-| | HDFS partitions=1831/1831 files=1831 size=280.96GB
+| | partition predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk IS NOT NULL
+| | HDFS partitions=1830/1831 files=1830 size=280.81GB
| | runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk
| | stored statistics:
| | table: rows=4.32G size=280.96GB
-| | partitions: 1831/1831 rows=4.32G
+| | partitions: 1830/1830 rows=4.30G
| | columns: all
-| | extrapolated-rows=disabled max-scan-range-rows=21.52M
-| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=12 row-size=8B cardinality=4.32G cost=497672983
+| | extrapolated-rows=disabled max-scan-range-rows=5.09M
+| | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| | tuple-ids=12 row-size=8B cardinality=4.30G cost=495194194
| | in pipelines: 10(GETNEXT)
| |
| 09:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
-| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| partition predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=145.67GB
| runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk
| stored statistics:
| table: rows=2.16G size=145.75GB
-| partitions: 1824/1824 rows=2.16G
+| partitions: 1823/1823 rows=2.16G
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=3.37M
| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=10 row-size=8B cardinality=2.16G cost=248828309
+| tuple-ids=10 row-size=8B cardinality=2.16G cost=248762936
| in pipelines: 09(GETNEXT)
|
07:HASH JOIN [INNER JOIN]
@@ -169,23 +173,23 @@ PLAN-ROOT SINK
|
|--06:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT))
+| predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_0.d_week_seq IS NOT NULL
| runtime filters: RF000[bloom] -> subtract(tpcds_partitioned_parquet_snap.date_dim_0.d_week_seq, 53)
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT))
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT))
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_0.d_week_seq IS NOT NULL
| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
-| tuple-ids=8 row-size=8B cardinality=373 cost=10467
+| tuple-ids=8 row-size=8B cardinality=373 cost=12520
| in pipelines: 06(GETNEXT)
|
05:AGGREGATE [FINALIZE]
| output: sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Sunday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Monday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Tuesday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Wednesday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_s [...]
| group by: tpcds_partitioned_parquet_snap.date_dim.d_week_seq
| mem-estimate=10.00MB mem-reservation=2.88MB spill-buffer=128.00KB thread-reservation=0
-| tuple-ids=7 row-size=116B cardinality=10.64K cost=4617060482
+| tuple-ids=7 row-size=116B cardinality=10.64K cost=4601325105
| in pipelines: 05(GETNEXT), 01(OPEN), 02(OPEN)
|
04:HASH JOIN [INNER JOIN]
@@ -193,50 +197,54 @@ PLAN-ROOT SINK
| fk/pk conjuncts: assumed fk/pk
| runtime filters: RF004[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| mem-estimate=4.75MB mem-reservation=4.75MB spill-buffer=256.00KB thread-reservation=0
-| tuple-ids=4,5 row-size=35B cardinality=6.48G cost=2836389159
+| tuple-ids=4,5 row-size=35B cardinality=6.46G cost=2826722669
| in pipelines: 01(GETNEXT), 02(GETNEXT), 03(OPEN)
|
|--03:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| HDFS partitions=1/1 files=1 size=2.17MB
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim.d_week_seq IS NOT NULL
| runtime filters: RF003[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF000[bloom] -> subtract(tpcds_partitioned_parquet_snap.date_dim.d_week_seq, 53), RF002[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim.d_week_seq IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=5 row-size=27B cardinality=73.05K cost=15928
+| tuple-ids=5 row-size=27B cardinality=73.05K cost=20034
| in pipelines: 03(GETNEXT)
|
00:UNION
| mem-estimate=0B mem-reservation=0B thread-reservation=0
-| tuple-ids=4 row-size=8B cardinality=6.48G cost=330871146
+| tuple-ids=4 row-size=8B cardinality=6.46G cost=329743499
| in pipelines: 01(GETNEXT), 02(GETNEXT)
|
|--02:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales]
-| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| partition predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk IS NOT NULL
+| HDFS partitions=1830/1831 files=1830 size=280.81GB
| runtime filters: RF004[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
| stored statistics:
| table: rows=4.32G size=280.96GB
-| partitions: 1831/1831 rows=4.32G
+| partitions: 1830/1830 rows=4.30G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=21.52M
-| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=2 row-size=8B cardinality=4.32G cost=497672983
+| extrapolated-rows=disabled max-scan-range-rows=5.09M
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=2 row-size=8B cardinality=4.30G cost=495194194
| in pipelines: 02(GETNEXT)
|
01:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
- HDFS partitions=1824/1824 files=1824 size=145.75GB
+ partition predicates: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=145.67GB
runtime filters: RF004[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
stored statistics:
table: rows=2.16G size=145.75GB
- partitions: 1824/1824 rows=2.16G
+ partitions: 1823/1823 rows=2.16G
columns: all
extrapolated-rows=disabled max-scan-range-rows=3.37M
mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
- tuple-ids=0 row-size=8B cardinality=2.16G cost=248828309
+ tuple-ids=0 row-size=8B cardinality=2.16G cost=248762936
in pipelines: 01(GETNEXT)
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=242.19MB Threads=37
+Max Per-Host Resource Reservation: Memory=224.19MB Threads=37
Per-Host Resource Estimates: Memory=1019MB
F13:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=6.32MB mem-reservation=4.00MB thread-reservation=1
@@ -309,18 +317,18 @@ max-parallelism=10 segment-costs=[29390, 16097] cpu-comparison-result=262 [max(1
| | |
| | F11:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.47MB mem-reservation=256.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10851]
+| | max-parallelism=1 segment-costs=[12904]
| | 14:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT), tpcds_partitioned_parquet_snap.date_dim_2.d_week_seq IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT), tpcds_partitioned_parquet_snap.date_dim_2.d_week_seq IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
-| | tuple-ids=18 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=18 row-size=8B cardinality=373 cost=12520
| | in pipelines: 14(GETNEXT)
| |
| 24:AGGREGATE [FINALIZE]
@@ -337,13 +345,13 @@ max-parallelism=10 segment-costs=[29390, 16097] cpu-comparison-result=262 [max(1
| |
| F08:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-| Per-Instance Resources: mem-estimate=30.69MB mem-reservation=4.00MB thread-reservation=1
-| max-parallelism=120 segment-costs=[8534050783, 10197930] cpu-comparison-result=120 [max(120 (self) vs 11 (sum children))]
+| Per-Instance Resources: mem-estimate=30.69MB mem-reservation=3.25MB thread-reservation=1
+| max-parallelism=120 segment-costs=[8504977107, 10197930] cpu-comparison-result=120 [max(120 (self) vs 11 (sum children))]
| 13:AGGREGATE [STREAMING]
| | output: sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Sunday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Monday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Tuesday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Wednesday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitione [...]
| | group by: tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq
| | mem-estimate=10.00MB mem-reservation=3.00MB spill-buffer=128.00KB thread-reservation=0
-| | tuple-ids=17 row-size=116B cardinality=1.28M cost=4620362235
+| | tuple-ids=17 row-size=116B cardinality=1.28M cost=4604626858
| | in pipelines: 09(GETNEXT), 10(GETNEXT)
| |
| 12:HASH JOIN [INNER JOIN, BROADCAST]
@@ -351,7 +359,7 @@ max-parallelism=10 segment-costs=[29390, 16097] cpu-comparison-result=262 [max(1
| | hash predicates: SOLD_DATE_SK = tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk
| | fk/pk conjuncts: assumed fk/pk
| | mem-estimate=0B mem-reservation=0B spill-buffer=256.00KB thread-reservation=0
-| | tuple-ids=14,15 row-size=35B cardinality=6.48G cost=2836316110
+| | tuple-ids=14,15 row-size=35B cardinality=6.46G cost=2826649620
| | in pipelines: 09(GETNEXT), 10(GETNEXT), 11(OPEN)
| |
| |--F16:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -371,45 +379,49 @@ max-parallelism=10 segment-costs=[29390, 16097] cpu-comparison-result=262 [max(1
| | F09:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
| | Per-Instance Resources: mem-estimate=16.12MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[22859]
+| | max-parallelism=1 segment-costs=[26965]
| | 11:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq IS NOT NULL
| | runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq, RF006[bloom] -> tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=15 row-size=27B cardinality=73.05K cost=15928
+| | tuple-ids=15 row-size=27B cardinality=73.05K cost=20034
| | in pipelines: 11(GETNEXT)
| |
| 08:UNION
| | mem-estimate=0B mem-reservation=0B thread-reservation=0
-| | tuple-ids=14 row-size=8B cardinality=6.48G cost=330871146
+| | tuple-ids=14 row-size=8B cardinality=6.46G cost=329743499
| | in pipelines: 09(GETNEXT), 10(GETNEXT)
| |
| |--10:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
-| | HDFS partitions=1831/1831 files=1831 size=280.96GB
+| | partition predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk IS NOT NULL
+| | HDFS partitions=1830/1831 files=1830 size=280.81GB
| | runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk
| | stored statistics:
| | table: rows=4.32G size=280.96GB
-| | partitions: 1831/1831 rows=4.32G
+| | partitions: 1830/1830 rows=4.30G
| | columns: all
-| | extrapolated-rows=disabled max-scan-range-rows=21.52M
-| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=12 row-size=8B cardinality=4.32G cost=497672983
+| | extrapolated-rows=disabled max-scan-range-rows=5.09M
+| | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| | tuple-ids=12 row-size=8B cardinality=4.30G cost=495194194
| | in pipelines: 10(GETNEXT)
| |
| 09:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
-| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| partition predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=145.67GB
| runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk
| stored statistics:
| table: rows=2.16G size=145.75GB
-| partitions: 1824/1824 rows=2.16G
+| partitions: 1823/1823 rows=2.16G
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=3.37M
| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=10 row-size=8B cardinality=2.16G cost=248828309
+| tuple-ids=10 row-size=8B cardinality=2.16G cost=248762936
| in pipelines: 09(GETNEXT)
|
26:EXCHANGE [HASH(subtract(tpcds_partitioned_parquet_snap.date_dim.d_week_seq, 53))]
@@ -445,19 +457,19 @@ max-parallelism=10 segment-costs=[1760943, 26273] cpu-comparison-result=131 [max
| F05:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
| Per-Instance Resources: mem-estimate=16.47MB mem-reservation=256.00KB thread-reservation=1
-| max-parallelism=1 segment-costs=[10851]
+| max-parallelism=1 segment-costs=[12904]
| 06:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT))
+| predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_0.d_week_seq IS NOT NULL
| runtime filters: RF000[bloom] -> subtract(tpcds_partitioned_parquet_snap.date_dim_0.d_week_seq, 53)
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT))
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT))
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_0.d_week_seq IS NOT NULL
| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
-| tuple-ids=8 row-size=8B cardinality=373 cost=10467
+| tuple-ids=8 row-size=8B cardinality=373 cost=12520
| in pipelines: 06(GETNEXT)
|
20:AGGREGATE [FINALIZE]
@@ -474,13 +486,13 @@ max-parallelism=10 segment-costs=[1760943, 26273] cpu-comparison-result=131 [max
|
F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-Per-Instance Resources: mem-estimate=30.69MB mem-reservation=4.00MB thread-reservation=1
-max-parallelism=120 segment-costs=[8534050783, 10197930] cpu-comparison-result=120 [max(120 (self) vs 11 (sum children))]
+Per-Instance Resources: mem-estimate=30.69MB mem-reservation=3.25MB thread-reservation=1
+max-parallelism=120 segment-costs=[8504977107, 10197930] cpu-comparison-result=120 [max(120 (self) vs 11 (sum children))]
05:AGGREGATE [STREAMING]
| output: sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Sunday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Monday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Tuesday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Wednesday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_s [...]
| group by: tpcds_partitioned_parquet_snap.date_dim.d_week_seq
| mem-estimate=10.00MB mem-reservation=3.00MB spill-buffer=128.00KB thread-reservation=0
-| tuple-ids=7 row-size=116B cardinality=1.28M cost=4620362235
+| tuple-ids=7 row-size=116B cardinality=1.28M cost=4604626858
| in pipelines: 01(GETNEXT), 02(GETNEXT)
|
04:HASH JOIN [INNER JOIN, BROADCAST]
@@ -488,7 +500,7 @@ max-parallelism=120 segment-costs=[8534050783, 10197930] cpu-comparison-result=1
| hash predicates: SOLD_DATE_SK = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| fk/pk conjuncts: assumed fk/pk
| mem-estimate=0B mem-reservation=0B spill-buffer=256.00KB thread-reservation=0
-| tuple-ids=4,5 row-size=35B cardinality=6.48G cost=2836316110
+| tuple-ids=4,5 row-size=35B cardinality=6.46G cost=2826649620
| in pipelines: 01(GETNEXT), 02(GETNEXT), 03(OPEN)
|
|--F18:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -508,48 +520,52 @@ max-parallelism=120 segment-costs=[8534050783, 10197930] cpu-comparison-result=1
| F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
| Per-Instance Resources: mem-estimate=16.12MB mem-reservation=512.00KB thread-reservation=1
-| max-parallelism=1 segment-costs=[22859]
+| max-parallelism=1 segment-costs=[26965]
| 03:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| HDFS partitions=1/1 files=1 size=2.17MB
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim.d_week_seq IS NOT NULL
| runtime filters: RF003[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF000[bloom] -> subtract(tpcds_partitioned_parquet_snap.date_dim.d_week_seq, 53), RF002[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim.d_week_seq IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=5 row-size=27B cardinality=73.05K cost=15928
+| tuple-ids=5 row-size=27B cardinality=73.05K cost=20034
| in pipelines: 03(GETNEXT)
|
00:UNION
| mem-estimate=0B mem-reservation=0B thread-reservation=0
-| tuple-ids=4 row-size=8B cardinality=6.48G cost=330871146
+| tuple-ids=4 row-size=8B cardinality=6.46G cost=329743499
| in pipelines: 01(GETNEXT), 02(GETNEXT)
|
|--02:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
-| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| partition predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk IS NOT NULL
+| HDFS partitions=1830/1831 files=1830 size=280.81GB
| runtime filters: RF004[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
| stored statistics:
| table: rows=4.32G size=280.96GB
-| partitions: 1831/1831 rows=4.32G
+| partitions: 1830/1830 rows=4.30G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=21.52M
-| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=2 row-size=8B cardinality=4.32G cost=497672983
+| extrapolated-rows=disabled max-scan-range-rows=5.09M
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=2 row-size=8B cardinality=4.30G cost=495194194
| in pipelines: 02(GETNEXT)
|
01:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
- HDFS partitions=1824/1824 files=1824 size=145.75GB
+ partition predicates: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=145.67GB
runtime filters: RF004[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
stored statistics:
table: rows=2.16G size=145.75GB
- partitions: 1824/1824 rows=2.16G
+ partitions: 1823/1823 rows=2.16G
columns: all
extrapolated-rows=disabled max-scan-range-rows=3.37M
mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
- tuple-ids=0 row-size=8B cardinality=2.16G cost=248828309
+ tuple-ids=0 row-size=8B cardinality=2.16G cost=248762936
in pipelines: 01(GETNEXT)
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=242.19MB Threads=37
+Max Per-Host Resource Reservation: Memory=224.19MB Threads=37
Per-Host Resource Estimates: Memory=1019MB
F13:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=6.32MB mem-reservation=4.00MB thread-reservation=1
@@ -622,18 +638,18 @@ max-parallelism=10 segment-costs=[29390, 16097] cpu-comparison-result=262 [max(1
| | |
| | F11:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.47MB mem-reservation=256.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10851]
+| | max-parallelism=1 segment-costs=[12904]
| | 14:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT), tpcds_partitioned_parquet_snap.date_dim_2.d_week_seq IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1998 AS INT), tpcds_partitioned_parquet_snap.date_dim_2.d_week_seq IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
-| | tuple-ids=18 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=18 row-size=8B cardinality=373 cost=12520
| | in pipelines: 14(GETNEXT)
| |
| 24:AGGREGATE [FINALIZE]
@@ -650,13 +666,13 @@ max-parallelism=10 segment-costs=[29390, 16097] cpu-comparison-result=262 [max(1
| |
| F08:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-| Per-Instance Resources: mem-estimate=30.69MB mem-reservation=4.00MB thread-reservation=1
-| max-parallelism=120 segment-costs=[8534050783, 10197930] cpu-comparison-result=120 [max(120 (self) vs 11 (sum children))]
+| Per-Instance Resources: mem-estimate=30.69MB mem-reservation=3.25MB thread-reservation=1
+| max-parallelism=120 segment-costs=[8504977107, 10197930] cpu-comparison-result=120 [max(120 (self) vs 11 (sum children))]
| 13:AGGREGATE [STREAMING]
| | output: sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Sunday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Monday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Tuesday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim_1.d_day_name = 'Wednesday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitione [...]
| | group by: tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq
| | mem-estimate=10.00MB mem-reservation=3.00MB spill-buffer=128.00KB thread-reservation=0
-| | tuple-ids=17 row-size=116B cardinality=1.28M cost=4620362235
+| | tuple-ids=17 row-size=116B cardinality=1.28M cost=4604626858
| | in pipelines: 09(GETNEXT), 10(GETNEXT)
| |
| 12:HASH JOIN [INNER JOIN, BROADCAST]
@@ -664,7 +680,7 @@ max-parallelism=10 segment-costs=[29390, 16097] cpu-comparison-result=262 [max(1
| | hash predicates: SOLD_DATE_SK = tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk
| | fk/pk conjuncts: assumed fk/pk
| | mem-estimate=0B mem-reservation=0B spill-buffer=256.00KB thread-reservation=0
-| | tuple-ids=14,15 row-size=35B cardinality=6.48G cost=2836316110
+| | tuple-ids=14,15 row-size=35B cardinality=6.46G cost=2826649620
| | in pipelines: 09(GETNEXT), 10(GETNEXT), 11(OPEN)
| |
| |--F16:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -684,45 +700,49 @@ max-parallelism=10 segment-costs=[29390, 16097] cpu-comparison-result=262 [max(1
| | F09:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
| | Per-Instance Resources: mem-estimate=16.12MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[22859]
+| | max-parallelism=1 segment-costs=[26965]
| | 11:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq IS NOT NULL
| | runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq, RF006[bloom] -> tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim_1.d_week_seq IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=15 row-size=27B cardinality=73.05K cost=15928
+| | tuple-ids=15 row-size=27B cardinality=73.05K cost=20034
| | in pipelines: 11(GETNEXT)
| |
| 08:UNION
| | mem-estimate=0B mem-reservation=0B thread-reservation=0
-| | tuple-ids=14 row-size=8B cardinality=6.48G cost=330871146
+| | tuple-ids=14 row-size=8B cardinality=6.46G cost=329743499
| | in pipelines: 09(GETNEXT), 10(GETNEXT)
| |
| |--10:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
-| | HDFS partitions=1831/1831 files=1831 size=280.96GB
+| | partition predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk IS NOT NULL
+| | HDFS partitions=1830/1831 files=1830 size=280.81GB
| | runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk
| | stored statistics:
| | table: rows=4.32G size=280.96GB
-| | partitions: 1831/1831 rows=4.32G
+| | partitions: 1830/1830 rows=4.30G
| | columns: all
-| | extrapolated-rows=disabled max-scan-range-rows=21.52M
-| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=12 row-size=8B cardinality=4.32G cost=497672983
+| | extrapolated-rows=disabled max-scan-range-rows=5.09M
+| | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| | tuple-ids=12 row-size=8B cardinality=4.30G cost=495194194
| | in pipelines: 10(GETNEXT)
| |
| 09:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
-| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| partition predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=145.67GB
| runtime filters: RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk
| stored statistics:
| table: rows=2.16G size=145.75GB
-| partitions: 1824/1824 rows=2.16G
+| partitions: 1823/1823 rows=2.16G
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=3.37M
| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=10 row-size=8B cardinality=2.16G cost=248828309
+| tuple-ids=10 row-size=8B cardinality=2.16G cost=248762936
| in pipelines: 09(GETNEXT)
|
26:EXCHANGE [HASH(subtract(tpcds_partitioned_parquet_snap.date_dim.d_week_seq, 53))]
@@ -758,19 +778,19 @@ max-parallelism=10 segment-costs=[1760943, 26273] cpu-comparison-result=131 [max
| F05:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
| Per-Instance Resources: mem-estimate=16.47MB mem-reservation=256.00KB thread-reservation=1
-| max-parallelism=1 segment-costs=[10851]
+| max-parallelism=1 segment-costs=[12904]
| 06:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT))
+| predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_0.d_week_seq IS NOT NULL
| runtime filters: RF000[bloom] -> subtract(tpcds_partitioned_parquet_snap.date_dim_0.d_week_seq, 53)
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT))
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT))
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1998 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_0.d_week_seq IS NOT NULL
| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
-| tuple-ids=8 row-size=8B cardinality=373 cost=10467
+| tuple-ids=8 row-size=8B cardinality=373 cost=12520
| in pipelines: 06(GETNEXT)
|
20:AGGREGATE [FINALIZE]
@@ -787,13 +807,13 @@ max-parallelism=10 segment-costs=[1760943, 26273] cpu-comparison-result=131 [max
|
F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-Per-Instance Resources: mem-estimate=30.69MB mem-reservation=4.00MB thread-reservation=1
-max-parallelism=120 segment-costs=[8534050783, 10197930] cpu-comparison-result=120 [max(120 (self) vs 11 (sum children))]
+Per-Instance Resources: mem-estimate=30.69MB mem-reservation=3.25MB thread-reservation=1
+max-parallelism=120 segment-costs=[8504977107, 10197930] cpu-comparison-result=120 [max(120 (self) vs 11 (sum children))]
05:AGGREGATE [STREAMING]
| output: sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Sunday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Monday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Tuesday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_snap.date_dim.d_day_name = 'Wednesday' THEN SALES_PRICE ELSE NULL END), sum(CASE WHEN tpcds_partitioned_parquet_s [...]
| group by: tpcds_partitioned_parquet_snap.date_dim.d_week_seq
| mem-estimate=10.00MB mem-reservation=3.00MB spill-buffer=128.00KB thread-reservation=0
-| tuple-ids=7 row-size=116B cardinality=1.28M cost=4620362235
+| tuple-ids=7 row-size=116B cardinality=1.28M cost=4604626858
| in pipelines: 01(GETNEXT), 02(GETNEXT)
|
04:HASH JOIN [INNER JOIN, BROADCAST]
@@ -801,7 +821,7 @@ max-parallelism=120 segment-costs=[8534050783, 10197930] cpu-comparison-result=1
| hash predicates: SOLD_DATE_SK = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| fk/pk conjuncts: assumed fk/pk
| mem-estimate=0B mem-reservation=0B spill-buffer=256.00KB thread-reservation=0
-| tuple-ids=4,5 row-size=35B cardinality=6.48G cost=2836316110
+| tuple-ids=4,5 row-size=35B cardinality=6.46G cost=2826649620
| in pipelines: 01(GETNEXT), 02(GETNEXT), 03(OPEN)
|
|--F18:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -821,44 +841,48 @@ max-parallelism=120 segment-costs=[8534050783, 10197930] cpu-comparison-result=1
| F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
| Per-Instance Resources: mem-estimate=16.12MB mem-reservation=512.00KB thread-reservation=1
-| max-parallelism=1 segment-costs=[22859]
+| max-parallelism=1 segment-costs=[26965]
| 03:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| HDFS partitions=1/1 files=1 size=2.17MB
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim.d_week_seq IS NOT NULL
| runtime filters: RF003[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq, RF000[bloom] -> subtract(tpcds_partitioned_parquet_snap.date_dim.d_week_seq, 53), RF002[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_week_seq
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim.d_week_seq IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=5 row-size=27B cardinality=73.05K cost=15928
+| tuple-ids=5 row-size=27B cardinality=73.05K cost=20034
| in pipelines: 03(GETNEXT)
|
00:UNION
| mem-estimate=0B mem-reservation=0B thread-reservation=0
-| tuple-ids=4 row-size=8B cardinality=6.48G cost=330871146
+| tuple-ids=4 row-size=8B cardinality=6.46G cost=329743499
| in pipelines: 01(GETNEXT), 02(GETNEXT)
|
|--02:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
-| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| partition predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk IS NOT NULL
+| HDFS partitions=1830/1831 files=1830 size=280.81GB
| runtime filters: RF004[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
| stored statistics:
| table: rows=4.32G size=280.96GB
-| partitions: 1831/1831 rows=4.32G
+| partitions: 1830/1830 rows=4.30G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=21.52M
-| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=2 row-size=8B cardinality=4.32G cost=497672983
+| extrapolated-rows=disabled max-scan-range-rows=5.09M
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=2 row-size=8B cardinality=4.30G cost=495194194
| in pipelines: 02(GETNEXT)
|
01:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
- HDFS partitions=1824/1824 files=1824 size=145.75GB
+ partition predicates: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=145.67GB
runtime filters: RF004[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
stored statistics:
table: rows=2.16G size=145.75GB
- partitions: 1824/1824 rows=2.16G
+ partitions: 1823/1823 rows=2.16G
columns: all
extrapolated-rows=disabled max-scan-range-rows=3.37M
mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
- tuple-ids=0 row-size=8B cardinality=2.16G cost=248828309
+ tuple-ids=0 row-size=8B cardinality=2.16G cost=248762936
in pipelines: 01(GETNEXT)
====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q03.test b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q03.test
index 85ccef606..4deb2be35 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q03.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q03.test
@@ -21,11 +21,11 @@ select dt.d_year
# end query 3 in stream 0 using template query3.tpl
---- PLAN
-Max Per-Host Resource Reservation: Memory=47.88MB Threads=1
-Per-Host Resource Estimates: Memory=543MB
+Max Per-Host Resource Reservation: Memory=40.38MB Threads=1
+Per-Host Resource Estimates: Memory=537MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Instance Resources: mem-estimate=543.48MB mem-reservation=47.88MB thread-reservation=1 runtime-filters-memory=2.00MB
-| max-parallelism=1 segment-costs=[3118055372, 117158429, 400]
+| Per-Instance Resources: mem-estimate=537.26MB mem-reservation=40.38MB thread-reservation=1 runtime-filters-memory=2.00MB
+| max-parallelism=1 segment-costs=[3209066006, 111544107, 400]
PLAN-ROOT SINK
| output exprs: tpcds_partitioned_parquet_snap.date_dim.d_year, tpcds_partitioned_parquet_snap.item.i_brand_id, tpcds_partitioned_parquet_snap.item.i_brand, sum(tpcds_partitioned_parquet_snap.store_sales.ss_sales_price)
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=400
@@ -33,14 +33,14 @@ PLAN-ROOT SINK
06:TOP-N [LIMIT=100]
| order by: tpcds_partitioned_parquet_snap.date_dim.d_year ASC NULLS LAST, sum(tpcds_partitioned_parquet_snap.store_sales.ss_sales_price) DESC NULLS FIRST, tpcds_partitioned_parquet_snap.item.i_brand_id ASC NULLS LAST
| mem-estimate=5.09KB mem-reservation=0B thread-reservation=0
-| tuple-ids=7 row-size=52B cardinality=100 cost=117158429
+| tuple-ids=7 row-size=52B cardinality=100 cost=111544107
| in pipelines: 06(GETNEXT), 05(OPEN)
|
05:AGGREGATE [FINALIZE]
| output: sum(tpcds_partitioned_parquet_snap.store_sales.ss_sales_price)
| group by: tpcds_partitioned_parquet_snap.date_dim.d_year, tpcds_partitioned_parquet_snap.item.i_brand, tpcds_partitioned_parquet_snap.item.i_brand_id
-| mem-estimate=521.60MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=6 row-size=52B cardinality=9.00M cost=53096454
+| mem-estimate=515.39MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=6 row-size=52B cardinality=8.59M cost=50698364
| in pipelines: 05(GETNEXT), 00(OPEN)
|
04:HASH JOIN [INNER JOIN]
@@ -48,20 +48,20 @@ PLAN-ROOT SINK
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk = tpcds_partitioned_parquet_snap.item.i_item_sk
| runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.item.i_item_sk, RF001[min_max] <- tpcds_partitioned_parquet_snap.item.i_item_sk
| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=0,2,4 row-size=64B cardinality=9.00M cost=3937676
+| tuple-ids=0,2,4 row-size=64B cardinality=8.59M cost=3759849
| in pipelines: 00(GETNEXT), 03(OPEN)
|
|--03:SCAN HDFS [tpcds_partitioned_parquet_snap.item]
| HDFS partitions=1/1 files=1 size=33.54MB
-| predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT)
+| predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT), tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
| stored statistics:
| table: rows=360.00K size=33.54MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=101.54K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT), tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=4 row-size=40B cardinality=375 cost=156093
+| tuple-ids=4 row-size=40B cardinality=375 cost=166209
| in pipelines: 03(GETNEXT)
|
02:HASH JOIN [INNER JOIN]
@@ -69,35 +69,38 @@ PLAN-ROOT SINK
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| runtime filters: RF002[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=0,2 row-size=24B cardinality=9.00M(filtered from 8.64G) cost=1567869673
+| tuple-ids=0,2 row-size=24B cardinality=8.59M(filtered from 8.25G) cost=1497057350
| in pipelines: 00(GETNEXT), 01(OPEN)
|
|--01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT)
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=2 row-size=12B cardinality=6.09K cost=14675
+| tuple-ids=2 row-size=12B cardinality=6.09K cost=16728
| in pipelines: 01(GETNEXT)
|
00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=0 row-size=12B cardinality=9.00M(filtered from 8.64G) cost=1492980801
+ extrapolated-rows=disabled max-scan-range-rows=12.22M
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=0 row-size=12B cardinality=8.59M(filtered from 8.25G) cost=1657367506
in pipelines: 00(GETNEXT)
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=279.50MB Threads=19
+Max Per-Host Resource Reservation: Memory=189.50MB Threads=19
Per-Host Resource Estimates: Memory=553MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=4.11MB mem-reservation=4.00MB thread-reservation=1
@@ -115,34 +118,34 @@ PLAN-ROOT SINK
|
F03:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.date_dim.d_year,tpcds_partitioned_parquet_snap.item.i_brand,tpcds_partitioned_parquet_snap.item.i_brand_id)] hosts=10 instances=20 (adjusted from 120)
Per-Instance Resources: mem-estimate=50.58MB mem-reservation=34.00MB thread-reservation=1
-max-parallelism=20 segment-costs=[56254141, 117158429, 168] cpu-comparison-result=120 [max(20 (self) vs 120 (sum children))]
+max-parallelism=20 segment-costs=[53713435, 111544107, 168] cpu-comparison-result=120 [max(20 (self) vs 120 (sum children))]
06:TOP-N [LIMIT=100]
| order by: tpcds_partitioned_parquet_snap.date_dim.d_year ASC NULLS LAST, sum(tpcds_partitioned_parquet_snap.store_sales.ss_sales_price) DESC NULLS FIRST, tpcds_partitioned_parquet_snap.item.i_brand_id ASC NULLS LAST
| mem-estimate=5.09KB mem-reservation=0B thread-reservation=0
-| tuple-ids=7 row-size=52B cardinality=100 cost=117158429
+| tuple-ids=7 row-size=52B cardinality=100 cost=111544107
| in pipelines: 06(GETNEXT), 10(OPEN)
|
10:AGGREGATE [FINALIZE]
| output: sum:merge(tpcds_partitioned_parquet_snap.store_sales.ss_sales_price)
| group by: tpcds_partitioned_parquet_snap.date_dim.d_year, tpcds_partitioned_parquet_snap.item.i_brand, tpcds_partitioned_parquet_snap.item.i_brand_id
| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=6 row-size=52B cardinality=9.00M cost=53096454
+| tuple-ids=6 row-size=52B cardinality=8.59M cost=50698364
| in pipelines: 10(GETNEXT), 00(OPEN)
|
09:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.date_dim.d_year,tpcds_partitioned_parquet_snap.item.i_brand,tpcds_partitioned_parquet_snap.item.i_brand_id)]
| mem-estimate=16.58MB mem-reservation=0B thread-reservation=0
-| tuple-ids=6 row-size=52B cardinality=9.00M cost=3157687
+| tuple-ids=6 row-size=52B cardinality=8.59M cost=3015071
| in pipelines: 00(GETNEXT)
|
F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
-Per-Instance Resources: mem-estimate=30.39MB mem-reservation=13.00MB thread-reservation=1
-max-parallelism=320 segment-costs=[3117878142, 34850578] cpu-comparison-result=120 [max(120 (self) vs 25 (sum children))]
+Per-Instance Resources: mem-estimate=30.39MB mem-reservation=5.50MB thread-reservation=1
+max-parallelism=330 segment-costs=[3208876607, 33276559] cpu-comparison-result=120 [max(120 (self) vs 25 (sum children))]
05:AGGREGATE [STREAMING]
| output: sum(tpcds_partitioned_parquet_snap.store_sales.ss_sales_price)
| group by: tpcds_partitioned_parquet_snap.date_dim.d_year, tpcds_partitioned_parquet_snap.item.i_brand, tpcds_partitioned_parquet_snap.item.i_brand_id
| mem-estimate=10.00MB mem-reservation=5.00MB spill-buffer=256.00KB thread-reservation=0
-| tuple-ids=6 row-size=52B cardinality=9.00M cost=53096454
+| tuple-ids=6 row-size=52B cardinality=8.59M cost=50698364
| in pipelines: 00(GETNEXT)
|
04:HASH JOIN [INNER JOIN, BROADCAST]
@@ -150,7 +153,7 @@ max-parallelism=320 segment-costs=[3117878142, 34850578] cpu-comparison-result=1
| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk = tpcds_partitioned_parquet_snap.item.i_item_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk = tpcds_partitioned_parquet_snap.item.i_item_sk
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=0,2,4 row-size=64B cardinality=9.00M cost=3937301
+| tuple-ids=0,2,4 row-size=64B cardinality=8.59M cost=3759474
| in pipelines: 00(GETNEXT), 03(OPEN)
|
|--F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -169,18 +172,18 @@ max-parallelism=320 segment-costs=[3117878142, 34850578] cpu-comparison-result=1
| |
| F02:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
| Per-Instance Resources: mem-estimate=16.17MB mem-reservation=512.00KB thread-reservation=1
-| max-parallelism=4 segment-costs=[156141]
+| max-parallelism=4 segment-costs=[166257]
| 03:SCAN HDFS [tpcds_partitioned_parquet_snap.item, RANDOM]
| HDFS partitions=1/1 files=1 size=33.54MB
-| predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT)
+| predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT), tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
| stored statistics:
| table: rows=360.00K size=33.54MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=101.54K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT), tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=4 row-size=40B cardinality=375 cost=156093
+| tuple-ids=4 row-size=40B cardinality=375 cost=166209
| in pipelines: 03(GETNEXT)
|
02:HASH JOIN [INNER JOIN, BROADCAST]
@@ -188,7 +191,7 @@ max-parallelism=320 segment-costs=[3117878142, 34850578] cpu-comparison-result=1
| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=0,2 row-size=24B cardinality=9.00M(filtered from 8.64G) cost=1567863586
+| tuple-ids=0,2 row-size=24B cardinality=8.59M(filtered from 8.25G) cost=1497051263
| in pipelines: 00(GETNEXT), 01(OPEN)
|
|--F06:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -207,33 +210,36 @@ max-parallelism=320 segment-costs=[3117878142, 34850578] cpu-comparison-result=1
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=16.06MB mem-reservation=512.00KB thread-reservation=1
-| max-parallelism=1 segment-costs=[15003]
+| max-parallelism=1 segment-costs=[17056]
| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT)
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=2 row-size=12B cardinality=6.09K cost=14675
+| tuple-ids=2 row-size=12B cardinality=6.09K cost=16728
| in pipelines: 01(GETNEXT)
|
00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=0 row-size=12B cardinality=9.00M(filtered from 8.64G) cost=1492980801
+ extrapolated-rows=disabled max-scan-range-rows=12.22M
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=0 row-size=12B cardinality=8.59M(filtered from 8.25G) cost=1657367506
in pipelines: 00(GETNEXT)
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=279.50MB Threads=19
+Max Per-Host Resource Reservation: Memory=189.50MB Threads=19
Per-Host Resource Estimates: Memory=553MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=4.11MB mem-reservation=4.00MB thread-reservation=1
@@ -251,34 +257,34 @@ PLAN-ROOT SINK
|
F03:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.date_dim.d_year,tpcds_partitioned_parquet_snap.item.i_brand,tpcds_partitioned_parquet_snap.item.i_brand_id)] hosts=10 instances=20 (adjusted from 120)
Per-Instance Resources: mem-estimate=50.58MB mem-reservation=34.00MB thread-reservation=1
-max-parallelism=20 segment-costs=[56254141, 117158429, 168] cpu-comparison-result=120 [max(20 (self) vs 120 (sum children))]
+max-parallelism=20 segment-costs=[53713435, 111544107, 168] cpu-comparison-result=120 [max(20 (self) vs 120 (sum children))]
06:TOP-N [LIMIT=100]
| order by: tpcds_partitioned_parquet_snap.date_dim.d_year ASC NULLS LAST, sum(tpcds_partitioned_parquet_snap.store_sales.ss_sales_price) DESC NULLS FIRST, tpcds_partitioned_parquet_snap.item.i_brand_id ASC NULLS LAST
| mem-estimate=5.09KB mem-reservation=0B thread-reservation=0
-| tuple-ids=7 row-size=52B cardinality=100 cost=117158429
+| tuple-ids=7 row-size=52B cardinality=100 cost=111544107
| in pipelines: 06(GETNEXT), 10(OPEN)
|
10:AGGREGATE [FINALIZE]
| output: sum:merge(tpcds_partitioned_parquet_snap.store_sales.ss_sales_price)
| group by: tpcds_partitioned_parquet_snap.date_dim.d_year, tpcds_partitioned_parquet_snap.item.i_brand, tpcds_partitioned_parquet_snap.item.i_brand_id
| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=6 row-size=52B cardinality=9.00M cost=53096454
+| tuple-ids=6 row-size=52B cardinality=8.59M cost=50698364
| in pipelines: 10(GETNEXT), 00(OPEN)
|
09:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.date_dim.d_year,tpcds_partitioned_parquet_snap.item.i_brand,tpcds_partitioned_parquet_snap.item.i_brand_id)]
| mem-estimate=16.58MB mem-reservation=0B thread-reservation=0
-| tuple-ids=6 row-size=52B cardinality=9.00M cost=3157687
+| tuple-ids=6 row-size=52B cardinality=8.59M cost=3015071
| in pipelines: 00(GETNEXT)
|
F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
-Per-Instance Resources: mem-estimate=30.39MB mem-reservation=13.00MB thread-reservation=1
-max-parallelism=320 segment-costs=[3117878142, 34850578] cpu-comparison-result=120 [max(120 (self) vs 25 (sum children))]
+Per-Instance Resources: mem-estimate=30.39MB mem-reservation=5.50MB thread-reservation=1
+max-parallelism=330 segment-costs=[3208876607, 33276559] cpu-comparison-result=120 [max(120 (self) vs 25 (sum children))]
05:AGGREGATE [STREAMING]
| output: sum(tpcds_partitioned_parquet_snap.store_sales.ss_sales_price)
| group by: tpcds_partitioned_parquet_snap.date_dim.d_year, tpcds_partitioned_parquet_snap.item.i_brand, tpcds_partitioned_parquet_snap.item.i_brand_id
| mem-estimate=10.00MB mem-reservation=5.00MB spill-buffer=256.00KB thread-reservation=0
-| tuple-ids=6 row-size=52B cardinality=9.00M cost=53096454
+| tuple-ids=6 row-size=52B cardinality=8.59M cost=50698364
| in pipelines: 00(GETNEXT)
|
04:HASH JOIN [INNER JOIN, BROADCAST]
@@ -286,7 +292,7 @@ max-parallelism=320 segment-costs=[3117878142, 34850578] cpu-comparison-result=1
| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk = tpcds_partitioned_parquet_snap.item.i_item_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk = tpcds_partitioned_parquet_snap.item.i_item_sk
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=0,2,4 row-size=64B cardinality=9.00M cost=3937301
+| tuple-ids=0,2,4 row-size=64B cardinality=8.59M cost=3759474
| in pipelines: 00(GETNEXT), 03(OPEN)
|
|--F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -305,18 +311,18 @@ max-parallelism=320 segment-costs=[3117878142, 34850578] cpu-comparison-result=1
| |
| F02:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
| Per-Instance Resources: mem-estimate=16.17MB mem-reservation=512.00KB thread-reservation=1
-| max-parallelism=4 segment-costs=[156141]
+| max-parallelism=4 segment-costs=[166257]
| 03:SCAN HDFS [tpcds_partitioned_parquet_snap.item, RANDOM]
| HDFS partitions=1/1 files=1 size=33.54MB
-| predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT)
+| predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT), tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
| stored statistics:
| table: rows=360.00K size=33.54MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=101.54K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.item.i_manufact_id = CAST(816 AS INT), tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=4 row-size=40B cardinality=375 cost=156093
+| tuple-ids=4 row-size=40B cardinality=375 cost=166209
| in pipelines: 03(GETNEXT)
|
02:HASH JOIN [INNER JOIN, BROADCAST]
@@ -324,7 +330,7 @@ max-parallelism=320 segment-costs=[3117878142, 34850578] cpu-comparison-result=1
| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=0,2 row-size=24B cardinality=9.00M(filtered from 8.64G) cost=1567863586
+| tuple-ids=0,2 row-size=24B cardinality=8.59M(filtered from 8.25G) cost=1497051263
| in pipelines: 00(GETNEXT), 01(OPEN)
|
|--F06:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -343,29 +349,32 @@ max-parallelism=320 segment-costs=[3117878142, 34850578] cpu-comparison-result=1
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=16.06MB mem-reservation=512.00KB thread-reservation=1
-| max-parallelism=1 segment-costs=[15003]
+| max-parallelism=1 segment-costs=[17056]
| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT)
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_moy = CAST(11 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=2 row-size=12B cardinality=6.09K cost=14675
+| tuple-ids=2 row-size=12B cardinality=6.09K cost=16728
| in pipelines: 01(GETNEXT)
|
00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=0 row-size=12B cardinality=9.00M(filtered from 8.64G) cost=1492980801
+ extrapolated-rows=disabled max-scan-range-rows=12.22M
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=0 row-size=12B cardinality=8.59M(filtered from 8.25G) cost=1657367506
in pipelines: 00(GETNEXT)
====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q04.test b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q04.test
index 703c338da..73d7ebc9c 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q04.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q04.test
@@ -116,11 +116,11 @@ limit 100
# end query 4 in stream 0 using template query4.tpl
---- PLAN
-Max Per-Host Resource Reservation: Memory=474.88MB Threads=1
-Per-Host Resource Estimates: Memory=76.67GB
+Max Per-Host Resource Reservation: Memory=491.38MB Threads=1
+Per-Host Resource Estimates: Memory=119.10GB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Instance Resources: mem-estimate=76.67GB mem-reservation=474.88MB thread-reservation=1 runtime-filters-memory=37.00MB
-| max-parallelism=1 segment-costs=[11764350338, 962284703, 567457869, 4555427905, 5889891799, 1209507461, 320028033, 400]
+| Per-Instance Resources: mem-estimate=119.10GB mem-reservation=491.38MB thread-reservation=1 runtime-filters-memory=70.00MB
+| max-parallelism=1 segment-costs=[10808104999, 5859651015, 3004714721, 3004714721, 10808104999, 5859651015, 2852181185, 400]
PLAN-ROOT SINK
| output exprs: tpcds_partitioned_parquet_snap.customer.c_customer_id, tpcds_partitioned_parquet_snap.customer.c_first_name, tpcds_partitioned_parquet_snap.customer.c_last_name, tpcds_partitioned_parquet_snap.customer.c_birth_country
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=400
@@ -128,248 +128,263 @@ PLAN-ROOT SINK
41:TOP-N [LIMIT=100]
| order by: tpcds_partitioned_parquet_snap.customer.c_customer_id ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_first_name ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_last_name ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_birth_country ASC NULLS LAST
| mem-estimate=8.27KB mem-reservation=0B thread-reservation=0
-| tuple-ids=42 row-size=85B cardinality=100 cost=50313918
+| tuple-ids=42 row-size=85B cardinality=100 cost=2263026260
| in pipelines: 41(GETNEXT), 23(OPEN)
|
40:HASH JOIN [INNER JOIN]
| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_4.c_customer_id
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_4.c_customer_id
-| other join predicates: CASE WHEN sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0)) THEN divide(sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_part [...]
-| runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_id, RF001[min_max] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_id
-| mem-estimate=736.18MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20,13,6,34,41 row-size=991B cardinality=4.06M cost=12496292
+| other join predicates: CASE WHEN sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0)) THEN divide(sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_part [...]
+| runtime filters: RF001[min_max] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_id
+| mem-estimate=14.55GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=27,20,13,34,6,41 row-size=991B cardinality=147.90M cost=144716495
| in pipelines: 23(GETNEXT), 39(OPEN)
|
|--39:AGGREGATE [FINALIZE]
-| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
+| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| | group by: tpcds_partitioned_parquet_snap.customer_4.c_customer_id, tpcds_partitioned_parquet_snap.customer_4.c_first_name, tpcds_partitioned_parquet_snap.customer_4.c_last_name, tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_4.c_birth_country, tpcds_partitioned_parquet_snap.customer_4.c_login, tpcds_partitioned_parquet_snap.customer_4.c_email_address
-| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
-| | mem-estimate=4.98GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=41 row-size=165B cardinality=4.06M cost=239926393
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
+| | mem-estimate=6.93GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=41 row-size=165B cardinality=79.98M cost=4721057758
| | in pipelines: 39(GETNEXT), 34(OPEN)
| |
| 38:HASH JOIN [INNER JOIN]
-| | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
| | runtime filters: RF030[bloom] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_sk, RF031[min_max] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
-| | mem-estimate=534.16MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=35,37,39 row-size=185B cardinality=40.65M cost=61349826
+| | mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=35,37,39 row-size=185B cardinality=799.83M cost=398416277
| | in pipelines: 34(GETNEXT), 37(OPEN)
| |
| |--37:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_4.c_customer_id)
+| | predicates: tpcds_partitioned_parquet_snap.customer_4.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_4.c_customer_sk IS NOT NULL
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_4.c_customer_id)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_4.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_4.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=39 row-size=153B cardinality=3.00M cost=30714228
+| | tuple-ids=39 row-size=153B cardinality=30.00M cost=31557228
| | in pipelines: 37(GETNEXT)
| |
| 36:HASH JOIN [INNER JOIN]
-| | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
+| | runtime filters: RF032[bloom] <- tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=35,37 row-size=32B cardinality=198.77M(filtered from 441.70M) cost=131021618
+| | tuple-ids=35,37 row-size=32B cardinality=871.30M cost=381367031
| | in pipelines: 34(GETNEXT), 35(OPEN)
| |
| |--35:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=37 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=37 row-size=8B cardinality=373 cost=12520
| | in pipelines: 35(GETNEXT)
| |
-| 34:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
-| HDFS partitions=1824/1824 files=1824 size=145.75GB
-| runtime filters: RF031[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk, RF030[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk
+| 34:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales]
+| partition predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk IS NOT NULL
+| HDFS partitions=1830/1831 files=1830 size=280.81GB
+| predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk IS NOT NULL
+| runtime filters: RF031[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk, RF032[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk, RF030[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk
| stored statistics:
-| table: rows=2.16G size=145.75GB
-| partitions: 1824/1824 rows=2.16G
+| table: rows=4.32G size=280.96GB
+| partitions: 1830/1830 rows=4.30G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=3.37M
-| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=35 row-size=24B cardinality=198.77M(filtered from 2.16G) cost=746484929
+| extrapolated-rows=disabled max-scan-range-rows=5.09M est-scan-range=373(filtered from 1830)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=35 row-size=24B cardinality=871.30M(filtered from 4.28G) cost=327240201
| in pipelines: 34(GETNEXT)
|
33:HASH JOIN [INNER JOIN]
-| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_3.c_customer_id
-| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_3.c_customer_id
-| other join predicates: CASE WHEN sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0)) THEN divide(sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_part [...]
-| runtime filters: RF003[min_max] <- tpcds_partitioned_parquet_snap.customer_3.c_customer_id
-| mem-estimate=14.68GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20,13,6,34 row-size=826B cardinality=30.00M cost=93918070
-| in pipelines: 23(GETNEXT), 32(OPEN)
-|
-|--32:AGGREGATE [FINALIZE]
-| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
-| | group by: tpcds_partitioned_parquet_snap.customer_3.c_customer_id, tpcds_partitioned_parquet_snap.customer_3.c_first_name, tpcds_partitioned_parquet_snap.customer_3.c_last_name, tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_3.c_birth_country, tpcds_partitioned_parquet_snap.customer_3.c_login, tpcds_partitioned_parquet_snap.customer_3.c_email_address
-| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
-| | mem-estimate=6.95GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=34 row-size=165B cardinality=80.79M cost=4768537611
-| | in pipelines: 32(GETNEXT), 27(OPEN)
-| |
-| 31:HASH JOIN [INNER JOIN]
-| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
-| | runtime filters: RF027[min_max] <- tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
-| | mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=28,30,32 row-size=185B cardinality=807.87M cost=402121453
-| | in pipelines: 27(GETNEXT), 30(OPEN)
-| |
-| |--30:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
-| | HDFS partitions=1/1 files=1 size=1.55GB
-| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_3.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer_3.c_customer_id
-| | stored statistics:
-| | table: rows=30.00M size=1.55GB
-| | columns: all
-| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=32 row-size=153B cardinality=30.00M cost=29871228
-| | in pipelines: 30(GETNEXT)
-| |
-| 29:HASH JOIN [INNER JOIN]
-| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
-| | runtime filters: RF028[bloom] <- tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
-| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=28,30 row-size=32B cardinality=880.06M cost=385202450
-| | in pipelines: 27(GETNEXT), 28(OPEN)
-| |
-| |--28:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
-| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT)
-| | stored statistics:
-| | table: rows=73.05K size=2.17MB
-| | columns: all
-| | extrapolated-rows=disabled max-scan-range-rows=73.05K
-| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT)
-| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=30 row-size=8B cardinality=373 cost=10467
-| | in pipelines: 28(GETNEXT)
-| |
-| 27:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales]
-| HDFS partitions=1831/1831 files=1831 size=280.96GB
-| runtime filters: RF027[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk, RF028[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk
-| stored statistics:
-| table: rows=4.32G size=280.96GB
-| partitions: 1831/1831 rows=4.32G
-| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=21.52M est-scan-range=374(filtered from 1831)
-| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=28 row-size=24B cardinality=880.06M(filtered from 4.32G) cost=304148590
-| in pipelines: 27(GETNEXT)
-|
-26:HASH JOIN [INNER JOIN]
| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer.c_customer_id
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer.c_customer_id
-| runtime filters: RF005[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_id
-| mem-estimate=5.36GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20,13,6 row-size=661B cardinality=30.00M cost=43131000
+| runtime filters: RF003[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_id
+| mem-estimate=27.50GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=27,20,13,34,6 row-size=826B cardinality=147.90M cost=185114797
| in pipelines: 23(GETNEXT), 05(OPEN)
|
|--05:AGGREGATE [FINALIZE]
| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales.ss_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| | group by: tpcds_partitioned_parquet_snap.customer.c_customer_id, tpcds_partitioned_parquet_snap.customer.c_first_name, tpcds_partitioned_parquet_snap.customer.c_last_name, tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer.c_birth_country, tpcds_partitioned_parquet_snap.customer.c_login, tpcds_partitioned_parquet_snap.customer.c_email_address
-| | mem-estimate=9.05GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=6 row-size=165B cardinality=30.00M cost=2364508786
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales.ss_ext_sales_price), 2)) IS NOT NULL
+| | mem-estimate=8.68GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=6 row-size=165B cardinality=147.90M cost=8729673496
| | in pipelines: 05(GETNEXT), 00(OPEN)
| |
| 04:HASH JOIN [INNER JOIN]
| | hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| | runtime filters: RF023[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | runtime filters: RF027[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk
| | mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=0,2,4 row-size=185B cardinality=1.62G cost=777080012
+| | tuple-ids=0,2,4 row-size=185B cardinality=1.48G cost=711235853
| | in pipelines: 00(GETNEXT), 03(OPEN)
| |
| |--03:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | predicates: tpcds_partitioned_parquet_snap.customer.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=4 row-size=153B cardinality=30.00M cost=29871228
+| | tuple-ids=4 row-size=153B cardinality=30.00M cost=31557228
| | in pipelines: 03(GETNEXT)
| |
| 02:HASH JOIN [INNER JOIN]
| | hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
-| | runtime filters: RF024[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
+| | runtime filters: RF028[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=0,2 row-size=32B cardinality=1.77G cost=773341361
+| | tuple-ids=0,2 row-size=32B cardinality=1.61G cost=705182682
| | in pipelines: 00(GETNEXT), 01(OPEN)
| |
| |--01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=2 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=2 row-size=8B cardinality=373 cost=12520
| | in pipelines: 01(GETNEXT)
| |
| 00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales]
-| HDFS partitions=1824/1824 files=1824 size=389.90GB
-| runtime filters: RF023[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF024[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk
+| partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=389.69GB
+| predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk IS NOT NULL
+| runtime filters: RF027[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF028[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk
| stored statistics:
| table: rows=8.64G size=389.90GB
-| partitions: 1824/1824 rows=8.64G
+| partitions: 1823/1823 rows=8.25G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
-| mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
-| tuple-ids=0 row-size=24B cardinality=1.77G(filtered from 8.64G) cost=610616051
+| extrapolated-rows=disabled max-scan-range-rows=12.22M est-scan-range=373(filtered from 1823)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=0 row-size=24B cardinality=1.61G(filtered from 7.88G) cost=630443220
| in pipelines: 00(GETNEXT)
|
+32:HASH JOIN [INNER JOIN]
+| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_3.c_customer_id
+| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_3.c_customer_id
+| runtime filters: RF005[min_max] <- tpcds_partitioned_parquet_snap.customer_3.c_customer_id
+| mem-estimate=7.16GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=27,20,13,34 row-size=661B cardinality=40.63M cost=58409061
+| in pipelines: 23(GETNEXT), 31(OPEN)
+|
+|--31:AGGREGATE [FINALIZE]
+| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
+| | group by: tpcds_partitioned_parquet_snap.customer_3.c_customer_id, tpcds_partitioned_parquet_snap.customer_3.c_first_name, tpcds_partitioned_parquet_snap.customer_3.c_last_name, tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_3.c_birth_country, tpcds_partitioned_parquet_snap.customer_3.c_login, tpcds_partitioned_parquet_snap.customer_3.c_email_address
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
+| | mem-estimate=5.92GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=34 row-size=165B cardinality=40.63M cost=2398033848
+| | in pipelines: 31(GETNEXT), 26(OPEN)
+| |
+| 30:HASH JOIN [INNER JOIN]
+| | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
+| | runtime filters: RF023[min_max] <- tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
+| | mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=28,30,32 row-size=185B cardinality=406.27M cost=216854633
+| | in pipelines: 26(GETNEXT), 29(OPEN)
+| |
+| |--29:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | predicates: tpcds_partitioned_parquet_snap.customer_3.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_3.c_customer_sk IS NOT NULL
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_3.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_3.c_customer_id
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_3.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_3.c_customer_sk IS NOT NULL
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=32 row-size=153B cardinality=30.00M cost=31557228
+| | in pipelines: 29(GETNEXT)
+| |
+| 28:HASH JOIN [INNER JOIN]
+| | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
+| | runtime filters: RF024[bloom] <- tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
+| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
+| | tuple-ids=28,30 row-size=32B cardinality=441.48M cost=193235083
+| | in pipelines: 26(GETNEXT), 27(OPEN)
+| |
+| |--27:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
+| | HDFS partitions=1/1 files=1 size=2.17MB
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk IS NOT NULL
+| | stored statistics:
+| | table: rows=73.05K size=2.17MB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=73.05K
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk IS NOT NULL
+| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| | tuple-ids=30 row-size=8B cardinality=373 cost=12520
+| | in pipelines: 27(GETNEXT)
+| |
+| 26:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
+| partition predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=145.67GB
+| predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk IS NOT NULL
+| runtime filters: RF023[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk, RF024[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk
+| stored statistics:
+| table: rows=2.16G size=145.75GB
+| partitions: 1823/1823 rows=2.16G
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=373(filtered from 1823)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=28 row-size=24B cardinality=441.48M(filtered from 2.16G) cost=165021409
+| in pipelines: 26(GETNEXT)
+|
25:HASH JOIN [INNER JOIN]
| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_0.c_customer_id
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_0.c_customer_id
-| runtime filters: RF007[min_max] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_id
-| mem-estimate=5.36GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20,13 row-size=495B cardinality=30.00M cost=43131000
+| runtime filters: RF006[bloom] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF007[min_max] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_id
+| mem-estimate=7.16GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=27,20,13 row-size=495B cardinality=40.63M cost=68503860
| in pipelines: 23(GETNEXT), 11(OPEN)
|
|--11:AGGREGATE [FINALIZE]
| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales.ws_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| | group by: tpcds_partitioned_parquet_snap.customer_0.c_customer_id, tpcds_partitioned_parquet_snap.customer_0.c_first_name, tpcds_partitioned_parquet_snap.customer_0.c_last_name, tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_0.c_birth_country, tpcds_partitioned_parquet_snap.customer_0.c_login, tpcds_partitioned_parquet_snap.customer_0.c_email_address
-| | mem-estimate=4.98GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=13 row-size=165B cardinality=30.00M cost=191708836
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales.ws_ext_sales_price), 2)) IS NOT NULL
+| | mem-estimate=5.92GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=13 row-size=165B cardinality=40.63M cost=2398033848
| | in pipelines: 11(GETNEXT), 06(OPEN)
| |
| 10:HASH JOIN [INNER JOIN]
| | hash predicates: tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
-| | runtime filters: RF018[bloom] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_sk, RF019[min_max] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
-| | mem-estimate=534.16MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=7,9,11 row-size=185B cardinality=40.65M cost=61349826
+| | runtime filters: RF019[min_max] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
+| | mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=7,9,11 row-size=185B cardinality=406.27M cost=216854633
| | in pipelines: 06(GETNEXT), 09(OPEN)
| |
| |--09:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_0.c_customer_id)
-| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id
+| | predicates: tpcds_partitioned_parquet_snap.customer_0.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_0.c_customer_sk IS NOT NULL
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_0.c_customer_id)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_0.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_0.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=11 row-size=153B cardinality=3.00M cost=30714228
+| | tuple-ids=11 row-size=153B cardinality=30.00M cost=31557228
| | in pipelines: 09(GETNEXT)
| |
| 08:HASH JOIN [INNER JOIN]
@@ -377,68 +392,72 @@ PLAN-ROOT SINK
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | runtime filters: RF020[bloom] <- tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=7,9 row-size=32B cardinality=198.77M(filtered from 441.70M) cost=131021618
+| | tuple-ids=7,9 row-size=32B cardinality=441.48M cost=193235083
| | in pipelines: 06(GETNEXT), 07(OPEN)
| |
| |--07:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=9 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=9 row-size=8B cardinality=373 cost=12520
| | in pipelines: 07(GETNEXT)
| |
| 06:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
-| HDFS partitions=1824/1824 files=1824 size=145.75GB
-| runtime filters: RF019[min_max] -> tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk, RF018[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk, RF020[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
+| partition predicates: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=145.67GB
+| predicates: tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk IS NOT NULL
+| runtime filters: RF019[min_max] -> tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk, RF020[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
| stored statistics:
| table: rows=2.16G size=145.75GB
-| partitions: 1824/1824 rows=2.16G
+| partitions: 1823/1823 rows=2.16G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=374(filtered from 1824)
+| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=373(filtered from 1823)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=7 row-size=24B cardinality=198.77M(filtered from 2.16G) cost=152652894
+| tuple-ids=7 row-size=24B cardinality=441.48M(filtered from 2.16G) cost=165021409
| in pipelines: 06(GETNEXT)
|
24:HASH JOIN [INNER JOIN]
| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_1.c_customer_id
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_1.c_customer_id
| runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF009[min_max] <- tpcds_partitioned_parquet_snap.customer_1.c_customer_id
-| mem-estimate=5.36GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20 row-size=330B cardinality=30.00M cost=77037753
+| mem-estimate=14.55GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=27,20 row-size=330B cardinality=79.98M cost=132410712
| in pipelines: 23(GETNEXT), 17(OPEN)
|
|--17:AGGREGATE [FINALIZE]
| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| | group by: tpcds_partitioned_parquet_snap.customer_1.c_customer_id, tpcds_partitioned_parquet_snap.customer_1.c_first_name, tpcds_partitioned_parquet_snap.customer_1.c_last_name, tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_1.c_birth_country, tpcds_partitioned_parquet_snap.customer_1.c_login, tpcds_partitioned_parquet_snap.customer_1.c_email_address
-| | mem-estimate=5.08GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=20 row-size=165B cardinality=30.00M cost=246864514
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_sales_price), 2)) IS NOT NULL
+| | mem-estimate=6.93GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=20 row-size=165B cardinality=79.98M cost=4721057758
| | in pipelines: 17(GETNEXT), 12(OPEN)
| |
| 16:HASH JOIN [INNER JOIN]
| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_1.c_customer_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_1.c_customer_sk
| | runtime filters: RF014[bloom] <- tpcds_partitioned_parquet_snap.customer_1.c_customer_sk, RF015[min_max] <- tpcds_partitioned_parquet_snap.customer_1.c_customer_sk
-| | mem-estimate=534.16MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=14,16,18 row-size=185B cardinality=80.79M cost=119359176
+| | mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=14,16,18 row-size=185B cardinality=799.83M cost=398416277
| | in pipelines: 12(GETNEXT), 15(OPEN)
| |
| |--15:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_1.c_customer_id)
-| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF007[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id
+| | predicates: tpcds_partitioned_parquet_snap.customer_1.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_1.c_customer_sk IS NOT NULL
+| | runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF006[bloom] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_1.c_customer_id)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_1.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_1.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=18 row-size=153B cardinality=3.00M cost=30714228
+| | tuple-ids=18 row-size=153B cardinality=30.00M cost=31557228
| | in pipelines: 15(GETNEXT)
| |
| 14:HASH JOIN [INNER JOIN]
@@ -446,40 +465,43 @@ PLAN-ROOT SINK
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk
| | runtime filters: RF016[bloom] <- tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk
| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=14,16 row-size=32B cardinality=396.57M(filtered from 880.06M) cost=261187728
+| | tuple-ids=14,16 row-size=32B cardinality=871.30M cost=381367031
| | in pipelines: 12(GETNEXT), 13(OPEN)
| |
| |--13:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=16 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=16 row-size=8B cardinality=373 cost=12520
| | in pipelines: 13(GETNEXT)
| |
| 12:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales]
-| HDFS partitions=1831/1831 files=1831 size=280.96GB
-| runtime filters: RF015[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk, RF014[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk, RF016[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
+| partition predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk IS NOT NULL
+| HDFS partitions=1830/1831 files=1830 size=280.81GB
+| predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk IS NOT NULL
+| runtime filters: RF015[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk, RF016[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk, RF014[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk
| stored statistics:
| table: rows=4.32G size=280.96GB
-| partitions: 1831/1831 rows=4.32G
+| partitions: 1830/1830 rows=4.30G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=21.52M est-scan-range=374(filtered from 1831)
-| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=14 row-size=24B cardinality=396.57M(filtered from 4.32G) cost=304148590
+| extrapolated-rows=disabled max-scan-range-rows=5.09M est-scan-range=373(filtered from 1830)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=14 row-size=24B cardinality=871.30M(filtered from 4.28G) cost=327240201
| in pipelines: 12(GETNEXT)
|
23:AGGREGATE [FINALIZE]
| output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| group by: tpcds_partitioned_parquet_snap.customer_2.c_customer_id, tpcds_partitioned_parquet_snap.customer_2.c_first_name, tpcds_partitioned_parquet_snap.customer_2.c_last_name, tpcds_partitioned_parquet_snap.customer_2.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_2.c_birth_country, tpcds_partitioned_parquet_snap.customer_2.c_login, tpcds_partitioned_parquet_snap.customer_2.c_email_address
| having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
-| mem-estimate=9.05GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27 row-size=165B cardinality=162.19M cost=9573431219
+| mem-estimate=8.68GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=27 row-size=165B cardinality=147.90M cost=8729673496
| in pipelines: 23(GETNEXT), 18(OPEN)
|
22:HASH JOIN [INNER JOIN]
@@ -487,18 +509,20 @@ PLAN-ROOT SINK
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk = tpcds_partitioned_parquet_snap.customer_2.c_customer_sk
| runtime filters: RF011[min_max] <- tpcds_partitioned_parquet_snap.customer_2.c_customer_sk
| mem-estimate=5.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=21,23,25 row-size=185B cardinality=1.62G cost=777080012
+| tuple-ids=21,23,25 row-size=185B cardinality=1.48G cost=711235853
| in pipelines: 18(GETNEXT), 21(OPEN)
|
|--21:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
| HDFS partitions=1/1 files=1 size=1.55GB
-| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF009[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF007[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF008[bloom] -> tpcds [...]
+| predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_2.c_customer_sk IS NOT NULL
+| runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF009[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF006[bloom] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF008[bloom] -> tpcds [...]
| stored statistics:
| table: rows=30.00M size=1.55GB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=3.10M
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_2.c_customer_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=25 row-size=153B cardinality=30.00M cost=29871228
+| tuple-ids=25 row-size=153B cardinality=30.00M cost=31557228
| in pipelines: 21(GETNEXT)
|
20:HASH JOIN [INNER JOIN]
@@ -506,618 +530,652 @@ PLAN-ROOT SINK
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales_0.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_2.d_date_sk
| runtime filters: RF012[bloom] <- tpcds_partitioned_parquet_snap.date_dim_2.d_date_sk
| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=21,23 row-size=32B cardinality=1.77G cost=773341361
+| tuple-ids=21,23 row-size=32B cardinality=1.61G cost=705182682
| in pipelines: 18(GETNEXT), 19(OPEN)
|
|--19:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT)
+| predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_2.d_date_sk IS NOT NULL
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_2.d_date_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=23 row-size=8B cardinality=373 cost=10467
+| tuple-ids=23 row-size=8B cardinality=373 cost=12520
| in pipelines: 19(GETNEXT)
|
18:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk IS NOT NULL
runtime filters: RF011[min_max] -> tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk, RF012[bloom] -> tpcds_partitioned_parquet_snap.store_sales_0.ss_sold_date_sk
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=21 row-size=24B cardinality=1.77G(filtered from 8.64G) cost=610616051
+ extrapolated-rows=disabled max-scan-range-rows=12.22M est-scan-range=373(filtered from 1823)
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=21 row-size=24B cardinality=1.61G(filtered from 7.88G) cost=630443220
in pipelines: 18(GETNEXT)
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=6.73GB Threads=219
-Per-Host Resource Estimates: Memory=131.02GB
-F28:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Instance Resources: mem-estimate=4.17MB mem-reservation=4.00MB thread-reservation=1
-| max-parallelism=1 segment-costs=[442] cpu-comparison-result=1020 [max(1 (self) vs 1020 (sum children))]
+Max Per-Host Resource Reservation: Memory=10.17GB Threads=379
+Per-Host Resource Estimates: Memory=184.29GB
+F31:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=5.02MB mem-reservation=4.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[442] cpu-comparison-result=1440 [max(1 (self) vs 1440 (sum children))]
PLAN-ROOT SINK
| output exprs: tpcds_partitioned_parquet_snap.customer.c_customer_id, tpcds_partitioned_parquet_snap.customer.c_first_name, tpcds_partitioned_parquet_snap.customer.c_last_name, tpcds_partitioned_parquet_snap.customer.c_birth_country
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=400
|
-75:MERGING-EXCHANGE [UNPARTITIONED]
+78:MERGING-EXCHANGE [UNPARTITIONED]
| order by: tpcds_partitioned_parquet_snap.customer.c_customer_id ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_first_name ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_last_name ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_birth_country ASC NULLS LAST
| limit: 100
-| mem-estimate=174.03KB mem-reservation=0B thread-reservation=0
+| mem-estimate=1.02MB mem-reservation=0B thread-reservation=0
| tuple-ids=42 row-size=85B cardinality=100 cost=42
| in pipelines: 41(GETNEXT)
|
-F09:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=20 (adjusted from 120)
+F10:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=120
Per-Instance Resources: mem-estimate=29.83MB mem-reservation=0B thread-reservation=1
-max-parallelism=30 segment-costs=[286406984, 253] cpu-comparison-result=1020 [max(20 (self) vs 1020 (sum children))]
+max-parallelism=260 segment-costs=[2591850306, 253] cpu-comparison-result=1440 [max(120 (self) vs 1440 (sum children))]
41:TOP-N [LIMIT=100]
| order by: tpcds_partitioned_parquet_snap.customer.c_customer_id ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_first_name ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_last_name ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_birth_country ASC NULLS LAST
| mem-estimate=8.27KB mem-reservation=0B thread-reservation=0
-| tuple-ids=42 row-size=85B cardinality=100 cost=50313918
+| tuple-ids=42 row-size=85B cardinality=100 cost=2263026260
| in pipelines: 41(GETNEXT), 46(OPEN)
|
40:HASH JOIN [INNER JOIN, PARTITIONED]
| hash-table-id=00
| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_4.c_customer_id
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_4.c_customer_id
-| other join predicates: CASE WHEN sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0)) THEN divide(sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_part [...]
+| other join predicates: CASE WHEN sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0)) THEN divide(sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_part [...]
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20,13,6,34,41 row-size=991B cardinality=4.06M cost=8431534
-| in pipelines: 46(GETNEXT), 73(OPEN)
+| tuple-ids=27,20,13,34,6,41 row-size=991B cardinality=147.90M cost=64733813
+| in pipelines: 46(GETNEXT), 76(OPEN)
|
-|--F29:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=20 (adjusted from 120)
-| | Per-Instance Resources: mem-estimate=55.76MB mem-reservation=38.00MB thread-reservation=1 runtime-filters-memory=4.00MB
-| | max-parallelism=30 segment-costs=[7604240] cpu-comparison-result=120 [max(20 (self) vs 120 (sum children))]
+|--F32:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=153.95MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=260 segment-costs=[149629463] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
| JOIN BUILD
| | join-table-id=00 plan-id=01 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.customer_4.c_customer_id
-| | runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_id, RF001[min_max] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_id
-| | mem-estimate=36.81MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=4064758
-| |
-| 74:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_id)]
-| | mem-estimate=14.96MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=41 row-size=165B cardinality=4.06M cost=3539482
-| | in pipelines: 73(GETNEXT)
-| |
-| F27:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_id,tpcds_partitioned_parquet_snap.customer_4.c_first_name,tpcds_partitioned_parquet_snap.customer_4.c_last_name,tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_4.c_birth_country,tpcds_partitioned_parquet_snap.customer_4.c_login,tpcds_partitioned_parquet_snap.customer_4.c_email_address)] hosts=10 instances=30 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=258.72MB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=30 segment-costs=[275321217, 45324595] cpu-comparison-result=120 [max(30 (self) vs 120 (sum children))]
-| 73:AGGREGATE [FINALIZE]
-| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), 2))
+| | runtime filters: RF001[min_max] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_id
+| | mem-estimate=124.13MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=79982682
+| |
+| 77:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_id)]
+| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=41 row-size=165B cardinality=79.98M cost=69646781
+| | in pipelines: 76(GETNEXT)
+| |
+| F30:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_id,tpcds_partitioned_parquet_snap.customer_4.c_first_name,tpcds_partitioned_parquet_snap.customer_4.c_last_name,tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_4.c_birth_country,tpcds_partitioned_parquet_snap.customer_4.c_login,tpcds_partitioned_parquet_snap.customer_4.c_email_address)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.18GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=550 segment-costs=[5417525566, 891856952] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 76:AGGREGATE [FINALIZE]
+| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2))
| | group by: tpcds_partitioned_parquet_snap.customer_4.c_customer_id, tpcds_partitioned_parquet_snap.customer_4.c_first_name, tpcds_partitioned_parquet_snap.customer_4.c_last_name, tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_4.c_birth_country, tpcds_partitioned_parquet_snap.customer_4.c_login, tpcds_partitioned_parquet_snap.customer_4.c_email_address
-| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
-| | mem-estimate=228.90MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=41 row-size=165B cardinality=4.06M cost=239926393
-| | in pipelines: 73(GETNEXT), 34(OPEN)
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
+| | mem-estimate=1.10GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=41 row-size=165B cardinality=79.98M cost=4721057758
+| | in pipelines: 76(GETNEXT), 34(OPEN)
| |
-| 72:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_id,tpcds_partitioned_parquet_snap.customer_4.c_first_name,tpcds_partitioned_parquet_snap.customer_4.c_last_name,tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_4.c_birth_country,tpcds_partitioned_parquet_snap.customer_4.c_login,tpcds_partitioned_parquet_snap.customer_4.c_email_address)]
+| 75:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_id,tpcds_partitioned_parquet_snap.customer_4.c_first_name,tpcds_partitioned_parquet_snap.customer_4.c_last_name,tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_4.c_birth_country,tpcds_partitioned_parquet_snap.customer_4.c_login,tpcds_partitioned_parquet_snap.customer_4.c_email_address)]
| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=41 row-size=165B cardinality=40.65M cost=35394824
+| | tuple-ids=41 row-size=165B cardinality=799.83M cost=696467808
| | in pipelines: 34(GETNEXT)
| |
-| F24:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
-| Per-Host Shared Resources: mem-estimate=4.00MB mem-reservation=4.00MB thread-reservation=0 runtime-filters-memory=4.00MB
-| Per-Instance Resources: mem-estimate=78.74MB mem-reservation=34.12MB thread-reservation=1
-| max-parallelism=120 segment-costs=[1175782393, 453245929] cpu-comparison-result=120 [max(120 (self) vs 31 (sum children))]
+| F29:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=938.49MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=900 segment-costs=[5346593754, 8918569464] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
| 39:AGGREGATE [STREAMING]
-| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
+| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| | group by: tpcds_partitioned_parquet_snap.customer_4.c_customer_id, tpcds_partitioned_parquet_snap.customer_4.c_first_name, tpcds_partitioned_parquet_snap.customer_4.c_last_name, tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_4.c_birth_country, tpcds_partitioned_parquet_snap.customer_4.c_login, tpcds_partitioned_parquet_snap.customer_4.c_email_address
-| | mem-estimate=42.92MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=41 row-size=165B cardinality=40.65M cost=239926393
+| | mem-estimate=844.52MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=41 row-size=165B cardinality=799.83M cost=4721057758
| | in pipelines: 34(GETNEXT)
| |
-| 38:HASH JOIN [INNER JOIN, BROADCAST]
+| 38:HASH JOIN [INNER JOIN, PARTITIONED]
| | hash-table-id=01
-| | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=35,37,39 row-size=185B cardinality=40.65M cost=58349826
+| | tuple-ids=35,37,39 row-size=185B cardinality=799.83M cost=368416277
| | in pipelines: 34(GETNEXT), 37(OPEN)
| |
-| |--F30:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=549.69MB mem-reservation=412.00MB thread-reservation=1 runtime-filters-memory=4.00MB
-| | | max-parallelism=10 segment-costs=[6987000]
+| |--F33:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=70.45MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | | max-parallelism=900 segment-costs=[54467197]
| | JOIN BUILD
| | | join-table-id=01 plan-id=02 cohort-id=02
| | | build expressions: tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
| | | runtime filters: RF030[bloom] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_sk, RF031[min_max] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
-| | | mem-estimate=534.16MB mem-reservation=408.00MB spill-buffer=2.00MB thread-reservation=0 cost=3000000
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
| | |
-| | 71:EXCHANGE [BROADCAST]
+| | 74:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_sk)]
| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=39 row-size=153B cardinality=3.00M cost=3987000
+| | | tuple-ids=39 row-size=153B cardinality=30.00M cost=24467197
| | | in pipelines: 37(GETNEXT)
| | |
-| | F26:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Instance Resources: mem-estimate=16.61MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=10 segment-costs=[32019513]
+| | F28:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[342891999]
| | 37:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_4.c_customer_id)
+| | predicates: tpcds_partitioned_parquet_snap.customer_4.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_4.c_customer_sk IS NOT NULL
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_4.c_customer_id)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_4.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_4.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=39 row-size=153B cardinality=3.00M cost=30714228
+| | tuple-ids=39 row-size=153B cardinality=30.00M cost=31557228
| | in pipelines: 37(GETNEXT)
| |
+| 73:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=35,37 row-size=32B cardinality=871.30M cost=257119719
+| | in pipelines: 34(GETNEXT)
+| |
+| F26:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=17.00MB mem-reservation=17.00MB thread-reservation=0 runtime-filters-memory=17.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=256.00KB thread-reservation=1
+| max-parallelism=350 segment-costs=[3401959876]
| 36:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=02
-| | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=35,37 row-size=32B cardinality=198.77M(filtered from 441.70M) cost=131021245
+| | tuple-ids=35,37 row-size=32B cardinality=871.30M cost=381366658
| | in pipelines: 34(GETNEXT), 35(OPEN)
| |
-| |--F31:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=23.27MB mem-reservation=23.25MB thread-reservation=1
+| |--F34:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
| | | join-table-id=02 plan-id=03 cohort-id=02
| | | build expressions: tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
+| | | runtime filters: RF032[bloom] <- tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
| | |
-| | 70:EXCHANGE [BROADCAST]
+| | 72:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
| | | tuple-ids=37 row-size=8B cardinality=373 cost=490
| | | in pipelines: 35(GETNEXT)
| | |
-| | F25:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | F27:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
+| | max-parallelism=1 segment-costs=[12536]
| | 35:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=37 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=37 row-size=8B cardinality=373 cost=12520
| | in pipelines: 35(GETNEXT)
| |
-| 34:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
-| HDFS partitions=1824/1824 files=1824 size=145.75GB
-| runtime filters: RF031[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk, RF030[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk
+| 34:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
+| partition predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk IS NOT NULL
+| HDFS partitions=1830/1831 files=1830 size=280.81GB
+| predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk IS NOT NULL
+| runtime filters: RF031[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk, RF032[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk, RF030[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk
| stored statistics:
-| table: rows=2.16G size=145.75GB
-| partitions: 1824/1824 rows=2.16G
+| table: rows=4.32G size=280.96GB
+| partitions: 1830/1830 rows=4.30G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=3.37M
-| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=35 row-size=24B cardinality=198.77M(filtered from 2.16G) cost=746484929
+| extrapolated-rows=disabled max-scan-range-rows=5.09M est-scan-range=373(filtered from 1830)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=35 row-size=24B cardinality=871.30M(filtered from 4.28G) cost=327240201
| in pipelines: 34(GETNEXT)
|
33:HASH JOIN [INNER JOIN, PARTITIONED]
| hash-table-id=03
-| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_3.c_customer_id
-| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_3.c_customer_id
-| other join predicates: CASE WHEN sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0)) THEN divide(sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_part [...]
+| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer.c_customer_id
+| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer.c_customer_id
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20,13,6,34 row-size=826B cardinality=30.00M cost=13131000
-| in pipelines: 46(GETNEXT), 68(OPEN)
+| tuple-ids=27,20,13,34,6 row-size=826B cardinality=147.90M cost=37219403
+| in pipelines: 46(GETNEXT), 70(OPEN)
|
-|--F32:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=20 (adjusted from 120)
-| | Per-Instance Resources: mem-estimate=781.55MB mem-reservation=34.00MB thread-reservation=1
-| | max-parallelism=30 segment-costs=[151134290] cpu-comparison-result=240 [max(20 (self) vs 240 (sum children))]
+|--F35:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=264.52MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=260 segment-costs=[276678749] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
| JOIN BUILD
| | join-table-id=03 plan-id=04 cohort-id=01
-| | build expressions: tpcds_partitioned_parquet_snap.customer_3.c_customer_id
-| | runtime filters: RF003[min_max] <- tpcds_partitioned_parquet_snap.customer_3.c_customer_id
-| | mem-estimate=751.73MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=80787070
+| | build expressions: tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | runtime filters: RF003[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | mem-estimate=234.70MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=147895394
| |
-| 69:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_id)]
+| 71:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_id)]
| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=34 row-size=165B cardinality=80.79M cost=70347220
-| | in pipelines: 68(GETNEXT)
+| | tuple-ids=6 row-size=165B cardinality=147.90M cost=128783355
+| | in pipelines: 70(GETNEXT)
| |
-| F23:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_id,tpcds_partitioned_parquet_snap.customer_3.c_first_name,tpcds_partitioned_parquet_snap.customer_3.c_last_name,tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_3.c_birth_country,tpcds_partitioned_parquet_snap.customer_3.c_login,tpcds_partitioned_parquet_snap.customer_3.c_email_address)] hosts=10 instances=120
-| Per-Instance Resources: mem-estimate=1.14GB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=550 segment-costs=[5472009822, 900826381] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
-| 68:AGGREGATE [FINALIZE]
-| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2))
-| | group by: tpcds_partitioned_parquet_snap.customer_3.c_customer_id, tpcds_partitioned_parquet_snap.customer_3.c_first_name, tpcds_partitioned_parquet_snap.customer_3.c_last_name, tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_3.c_birth_country, tpcds_partitioned_parquet_snap.customer_3.c_login, tpcds_partitioned_parquet_snap.customer_3.c_email_address
-| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
-| | mem-estimate=1.11GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=34 row-size=165B cardinality=80.79M cost=4768537611
-| | in pipelines: 68(GETNEXT), 27(OPEN)
+| F25:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_id,tpcds_partitioned_parquet_snap.customer.c_first_name,tpcds_partitioned_parquet_snap.customer.c_last_name,tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer.c_birth_country,tpcds_partitioned_parquet_snap.customer.c_login,tpcds_partitioned_parquet_snap.customer.c_email_address)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=2.11GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=1010 segment-costs=[10017507045, 1649126185] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 70:AGGREGATE [FINALIZE]
+| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales.ss_ext_sales_price), 2))
+| | group by: tpcds_partitioned_parquet_snap.customer.c_customer_id, tpcds_partitioned_parquet_snap.customer.c_first_name, tpcds_partitioned_parquet_snap.customer.c_last_name, tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer.c_birth_country, tpcds_partitioned_parquet_snap.customer.c_login, tpcds_partitioned_parquet_snap.customer.c_email_address
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales.ss_ext_sales_price), 2)) IS NOT NULL
+| | mem-estimate=2.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=6 row-size=165B cardinality=147.90M cost=8729673496
+| | in pipelines: 70(GETNEXT), 00(OPEN)
| |
-| 67:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_id,tpcds_partitioned_parquet_snap.customer_3.c_first_name,tpcds_partitioned_parquet_snap.customer_3.c_last_name,tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_3.c_birth_country,tpcds_partitioned_parquet_snap.customer_3.c_login,tpcds_partitioned_parquet_snap.customer_3.c_email_address)]
+| 69:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_id,tpcds_partitioned_parquet_snap.customer.c_first_name,tpcds_partitioned_parquet_snap.customer.c_last_name,tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer.c_birth_country,tpcds_partitioned_parquet_snap.customer.c_login,tpcds_partitioned_parquet_snap.customer.c_email_address)]
| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=34 row-size=165B cardinality=807.87M cost=703472211
-| | in pipelines: 27(GETNEXT)
+| | tuple-ids=6 row-size=165B cardinality=1.48G cost=1287833549
+| | in pipelines: 00(GETNEXT)
| |
-| F22:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk)] hosts=10 instances=120
-| Per-Instance Resources: mem-estimate=946.99MB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=910 segment-costs=[5400364645, 9008263849] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
-| 32:AGGREGATE [STREAMING]
-| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
-| | group by: tpcds_partitioned_parquet_snap.customer_3.c_customer_id, tpcds_partitioned_parquet_snap.customer_3.c_first_name, tpcds_partitioned_parquet_snap.customer_3.c_last_name, tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_3.c_birth_country, tpcds_partitioned_parquet_snap.customer_3.c_login, tpcds_partitioned_parquet_snap.customer_3.c_email_address
-| | mem-estimate=853.01MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=34 row-size=165B cardinality=807.87M cost=4768537611
-| | in pipelines: 27(GETNEXT)
+| F24:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.62GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=1650 segment-costs=[9886347547, 16491261804] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 05:AGGREGATE [STREAMING]
+| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales.ss_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
+| | group by: tpcds_partitioned_parquet_snap.customer.c_customer_id, tpcds_partitioned_parquet_snap.customer.c_first_name, tpcds_partitioned_parquet_snap.customer.c_last_name, tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer.c_birth_country, tpcds_partitioned_parquet_snap.customer.c_login, tpcds_partitioned_parquet_snap.customer.c_email_address
+| | mem-estimate=1.52GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=6 row-size=165B cardinality=1.48G cost=8729673496
+| | in pipelines: 00(GETNEXT)
| |
-| 31:HASH JOIN [INNER JOIN, PARTITIONED]
+| 04:HASH JOIN [INNER JOIN, PARTITIONED]
| | hash-table-id=04
-| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=28,30,32 row-size=185B cardinality=807.87M cost=372121453
-| | in pipelines: 27(GETNEXT), 30(OPEN)
+| | tuple-ids=0,2,4 row-size=185B cardinality=1.48G cost=681235853
+| | in pipelines: 00(GETNEXT), 03(OPEN)
| |
-| |--F33:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk)] hosts=10 instances=120
+| |--F36:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=120
| | | Per-Instance Resources: mem-estimate=54.45MB mem-reservation=34.00MB thread-reservation=1
-| | | max-parallelism=910 segment-costs=[54467197]
+| | | max-parallelism=1650 segment-costs=[54467197]
| | JOIN BUILD
| | | join-table-id=04 plan-id=05 cohort-id=03
-| | | build expressions: tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
-| | | runtime filters: RF027[min_max] <- tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
+| | | build expressions: tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | | runtime filters: RF027[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk
| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
| | |
-| | 66:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_sk)]
+| | 68:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_sk)]
| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=32 row-size=153B cardinality=30.00M cost=24467197
-| | | in pipelines: 30(GETNEXT)
+| | | tuple-ids=4 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 03(GETNEXT)
| | |
-| | F21:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Host Shared Resources: mem-estimate=4.00MB mem-reservation=4.00MB thread-reservation=0 runtime-filters-memory=4.00MB
+| | F23:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=10 segment-costs=[341205999]
-| | 30:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | max-parallelism=10 segment-costs=[342891999]
+| | 03:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_3.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer_3.c_customer_id
+| | predicates: tpcds_partitioned_parquet_snap.customer.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=32 row-size=153B cardinality=30.00M cost=29871228
-| | in pipelines: 30(GETNEXT)
+| | tuple-ids=4 row-size=153B cardinality=30.00M cost=31557228
+| | in pipelines: 03(GETNEXT)
| |
-| 65:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk)]
+| 67:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)]
| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=28,30 row-size=32B cardinality=880.06M cost=259705581
-| | in pipelines: 27(GETNEXT)
+| | tuple-ids=0,2 row-size=32B cardinality=1.61G cost=475438198
+| | in pipelines: 00(GETNEXT)
| |
-| F19:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| F21:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=1.00MB thread-reservation=1
-| max-parallelism=350 segment-costs=[3409790834]
-| 29:HASH JOIN [INNER JOIN, BROADCAST]
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=512.00KB thread-reservation=1
+| max-parallelism=373 segment-costs=[6315884965]
+| 02:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=05
-| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=28,30 row-size=32B cardinality=880.06M cost=385202077
-| | in pipelines: 27(GETNEXT), 28(OPEN)
+| | tuple-ids=0,2 row-size=32B cardinality=1.61G cost=705182309
+| | in pipelines: 00(GETNEXT), 01(OPEN)
| |
-| |--F34:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| |--F37:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
| | | join-table-id=05 plan-id=06 cohort-id=03
-| | | build expressions: tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
-| | | runtime filters: RF028[bloom] <- tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
+| | | build expressions: tpcds_partitioned_parquet_snap.date_dim.d_date_sk
+| | | runtime filters: RF028[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
| | |
-| | 64:EXCHANGE [BROADCAST]
+| | 66:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=30 row-size=8B cardinality=373 cost=490
-| | | in pipelines: 28(GETNEXT)
+| | | tuple-ids=2 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 01(GETNEXT)
| | |
-| | F20:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | F22:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
-| | 28:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | max-parallelism=1 segment-costs=[12536]
+| | 01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
-| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT)
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=30 row-size=8B cardinality=373 cost=10467
-| | in pipelines: 28(GETNEXT)
+| | tuple-ids=2 row-size=8B cardinality=373 cost=12520
+| | in pipelines: 01(GETNEXT)
| |
-| 27:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
-| HDFS partitions=1831/1831 files=1831 size=280.96GB
-| runtime filters: RF027[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk, RF028[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk
+| 00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+| partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=389.69GB
+| predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk IS NOT NULL
+| runtime filters: RF027[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF028[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk
| stored statistics:
-| table: rows=4.32G size=280.96GB
-| partitions: 1831/1831 rows=4.32G
+| table: rows=8.64G size=389.90GB
+| partitions: 1823/1823 rows=8.25G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=21.52M est-scan-range=374(filtered from 1831)
-| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=28 row-size=24B cardinality=880.06M(filtered from 4.32G) cost=304148590
-| in pipelines: 27(GETNEXT)
+| extrapolated-rows=disabled max-scan-range-rows=12.22M est-scan-range=373(filtered from 1823)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=0 row-size=24B cardinality=1.61G(filtered from 7.88G) cost=630443220
+| in pipelines: 00(GETNEXT)
|
-26:HASH JOIN [INNER JOIN, PARTITIONED]
+32:HASH JOIN [INNER JOIN, PARTITIONED]
| hash-table-id=06
-| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer.c_customer_id
-| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer.c_customer_id
+| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_3.c_customer_id
+| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_3.c_customer_id
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20,13,6 row-size=661B cardinality=30.00M cost=13131000
-| in pipelines: 46(GETNEXT), 62(OPEN)
+| tuple-ids=27,20,13,34 row-size=661B cardinality=40.63M cost=17782323
+| in pipelines: 46(GETNEXT), 64(OPEN)
|
-|--F35:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=20 (adjusted from 120)
-| | Per-Instance Resources: mem-estimate=304.47MB mem-reservation=34.00MB thread-reservation=1
-| | max-parallelism=30 segment-costs=[56123197] cpu-comparison-result=240 [max(20 (self) vs 240 (sum children))]
+|--F38:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=90.89MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=260 segment-costs=[76003415] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
| JOIN BUILD
| | join-table-id=06 plan-id=07 cohort-id=01
-| | build expressions: tpcds_partitioned_parquet_snap.customer.c_customer_id
-| | runtime filters: RF005[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_id
-| | mem-estimate=274.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | build expressions: tpcds_partitioned_parquet_snap.customer_3.c_customer_id
+| | runtime filters: RF005[min_max] <- tpcds_partitioned_parquet_snap.customer_3.c_customer_id
+| | mem-estimate=61.07MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=40626738
| |
-| 63:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_id)]
+| 65:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_id)]
| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=6 row-size=165B cardinality=30.00M cost=26123197
-| | in pipelines: 62(GETNEXT)
+| | tuple-ids=34 row-size=165B cardinality=40.63M cost=35376677
+| | in pipelines: 64(GETNEXT)
| |
-| F18:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_id,tpcds_partitioned_parquet_snap.customer.c_first_name,tpcds_partitioned_parquet_snap.customer.c_last_name,tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer.c_birth_country,tpcds_partitioned_parquet_snap.customer.c_login,tpcds_partitioned_parquet_snap.customer.c_email_address)] hosts=10 instances=120
-| Per-Instance Resources: mem-estimate=1.82GB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=310 segment-costs=[3067099075, 334518771] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
-| 62:AGGREGATE [FINALIZE]
-| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales.ss_ext_sales_price), 2))
-| | group by: tpcds_partitioned_parquet_snap.customer.c_customer_id, tpcds_partitioned_parquet_snap.customer.c_first_name, tpcds_partitioned_parquet_snap.customer.c_last_name, tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer.c_birth_country, tpcds_partitioned_parquet_snap.customer.c_login, tpcds_partitioned_parquet_snap.customer.c_email_address
-| | mem-estimate=1.80GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=6 row-size=165B cardinality=30.00M cost=1930086569
-| | in pipelines: 62(GETNEXT), 00(OPEN)
+| F20:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_id,tpcds_partitioned_parquet_snap.customer_3.c_first_name,tpcds_partitioned_parquet_snap.customer_3.c_last_name,tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_3.c_birth_country,tpcds_partitioned_parquet_snap.customer_3.c_login,tpcds_partitioned_parquet_snap.customer_3.c_email_address)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=651.24MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=280 segment-costs=[2751800623, 453013550] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 64:AGGREGATE [FINALIZE]
+| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), 2))
+| | group by: tpcds_partitioned_parquet_snap.customer_3.c_customer_id, tpcds_partitioned_parquet_snap.customer_3.c_first_name, tpcds_partitioned_parquet_snap.customer_3.c_last_name, tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_3.c_birth_country, tpcds_partitioned_parquet_snap.customer_3.c_login, tpcds_partitioned_parquet_snap.customer_3.c_email_address
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
+| | mem-estimate=571.96MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=34 row-size=165B cardinality=40.63M cost=2398033848
+| | in pipelines: 64(GETNEXT), 26(OPEN)
| |
-| 61:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_id,tpcds_partitioned_parquet_snap.customer.c_first_name,tpcds_partitioned_parquet_snap.customer.c_last_name,tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer.c_birth_country,tpcds_partitioned_parquet_snap.customer.c_login,tpcds_partitioned_parquet_snap.customer.c_email_address)]
+| 63:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_id,tpcds_partitioned_parquet_snap.customer_3.c_first_name,tpcds_partitioned_parquet_snap.customer_3.c_last_name,tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_3.c_birth_country,tpcds_partitioned_parquet_snap.customer_3.c_login,tpcds_partitioned_parquet_snap.customer_3.c_email_address)]
| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=6 row-size=165B cardinality=1.31G cost=1137012506
-| | in pipelines: 00(GETNEXT)
+| | tuple-ids=34 row-size=165B cardinality=406.27M cost=353766775
+| | in pipelines: 26(GETNEXT)
| |
-| F17:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=120
-| Per-Instance Resources: mem-estimate=1.76GB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=1460 segment-costs=[9410215497, 14559933564] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
-| 05:AGGREGATE [STREAMING]
-| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales.ss_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
-| | group by: tpcds_partitioned_parquet_snap.customer.c_customer_id, tpcds_partitioned_parquet_snap.customer.c_first_name, tpcds_partitioned_parquet_snap.customer.c_last_name, tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer.c_birth_country, tpcds_partitioned_parquet_snap.customer.c_login, tpcds_partitioned_parquet_snap.customer.c_email_address
-| | mem-estimate=1.67GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=6 row-size=165B cardinality=1.31G cost=8141744291
-| | in pipelines: 00(GETNEXT)
+| F19:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=522.94MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=460 segment-costs=[2715168496, 4530135523] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 31:AGGREGATE [STREAMING]
+| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
+| | group by: tpcds_partitioned_parquet_snap.customer_3.c_customer_id, tpcds_partitioned_parquet_snap.customer_3.c_first_name, tpcds_partitioned_parquet_snap.customer_3.c_last_name, tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_3.c_birth_country, tpcds_partitioned_parquet_snap.customer_3.c_login, tpcds_partitioned_parquet_snap.customer_3.c_email_address
+| | mem-estimate=428.97MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=34 row-size=165B cardinality=406.27M cost=2398033848
+| | in pipelines: 26(GETNEXT)
| |
-| 04:HASH JOIN [INNER JOIN, PARTITIONED]
+| 30:HASH JOIN [INNER JOIN, PARTITIONED]
| | hash-table-id=07
-| | hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=0,2,4 row-size=185B cardinality=1.62G cost=747080012
-| | in pipelines: 00(GETNEXT), 03(OPEN)
+| | tuple-ids=28,30,32 row-size=185B cardinality=406.27M cost=186854633
+| | in pipelines: 26(GETNEXT), 29(OPEN)
| |
-| |--F36:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=120
+| |--F39:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk)] hosts=10 instances=120
| | | Per-Instance Resources: mem-estimate=54.45MB mem-reservation=34.00MB thread-reservation=1
-| | | max-parallelism=1460 segment-costs=[54467197]
+| | | max-parallelism=460 segment-costs=[54467197]
| | JOIN BUILD
| | | join-table-id=07 plan-id=08 cohort-id=04
-| | | build expressions: tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| | | runtime filters: RF023[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | | build expressions: tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
+| | | runtime filters: RF023[min_max] <- tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
| | |
-| | 60:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_sk)]
+| | 62:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_sk)]
| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=4 row-size=153B cardinality=30.00M cost=24467197
-| | | in pipelines: 03(GETNEXT)
+| | | tuple-ids=32 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 29(GETNEXT)
| | |
-| | F16:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Host Shared Resources: mem-estimate=4.00MB mem-reservation=4.00MB thread-reservation=0 runtime-filters-memory=4.00MB
+| | F18:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=10 segment-costs=[341205999]
-| | 03:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | max-parallelism=10 segment-costs=[342891999]
+| | 29:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | predicates: tpcds_partitioned_parquet_snap.customer_3.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_3.c_customer_sk IS NOT NULL
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_3.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_3.c_customer_id
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_3.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_3.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=4 row-size=153B cardinality=30.00M cost=29871228
-| | in pipelines: 03(GETNEXT)
+| | tuple-ids=32 row-size=153B cardinality=30.00M cost=31557228
+| | in pipelines: 29(GETNEXT)
| |
-| 59:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)]
+| 61:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk)]
| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=0,2 row-size=32B cardinality=1.77G cost=521391194
-| | in pipelines: 00(GETNEXT)
+| | tuple-ids=28,30 row-size=32B cardinality=441.48M cost=130280015
+| | in pipelines: 26(GETNEXT)
| |
-| F14:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| F16:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=8.00MB thread-reservation=1
-| max-parallelism=374 segment-costs=[6845578389]
-| 02:HASH JOIN [INNER JOIN, BROADCAST]
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=180 segment-costs=[1722951425]
+| 28:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=08
-| | hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=0,2 row-size=32B cardinality=1.77G cost=773340988
-| | in pipelines: 00(GETNEXT), 01(OPEN)
+| | tuple-ids=28,30 row-size=32B cardinality=441.48M cost=193234710
+| | in pipelines: 26(GETNEXT), 27(OPEN)
| |
-| |--F37:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| |--F40:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
| | | join-table-id=08 plan-id=09 cohort-id=04
-| | | build expressions: tpcds_partitioned_parquet_snap.date_dim.d_date_sk
-| | | runtime filters: RF024[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
+| | | build expressions: tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
+| | | runtime filters: RF024[bloom] <- tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
| | |
-| | 58:EXCHANGE [BROADCAST]
+| | 60:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=2 row-size=8B cardinality=373 cost=490
-| | | in pipelines: 01(GETNEXT)
+| | | tuple-ids=30 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 27(GETNEXT)
| | |
-| | F15:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | F17:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
-| | 01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | max-parallelism=1 segment-costs=[12536]
+| | 27:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
-| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=2 row-size=8B cardinality=373 cost=10467
-| | in pipelines: 01(GETNEXT)
-| |
-| 00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
-| HDFS partitions=1824/1824 files=1824 size=389.90GB
-| runtime filters: RF023[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF024[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk
+| | tuple-ids=30 row-size=8B cardinality=373 cost=12520
+| | in pipelines: 27(GETNEXT)
+| |
+| 26:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
+| partition predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=145.67GB
+| predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk IS NOT NULL
+| runtime filters: RF023[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk, RF024[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk
| stored statistics:
-| table: rows=8.64G size=389.90GB
-| partitions: 1824/1824 rows=8.64G
+| table: rows=2.16G size=145.75GB
+| partitions: 1823/1823 rows=2.16G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
-| mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
-| tuple-ids=0 row-size=24B cardinality=1.77G(filtered from 8.64G) cost=610616051
-| in pipelines: 00(GETNEXT)
+| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=373(filtered from 1823)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=28 row-size=24B cardinality=441.48M(filtered from 2.16G) cost=165021409
+| in pipelines: 26(GETNEXT)
|
25:HASH JOIN [INNER JOIN, PARTITIONED]
| hash-table-id=09
| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_0.c_customer_id
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_0.c_customer_id
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20,13 row-size=495B cardinality=30.00M cost=13131000
-| in pipelines: 46(GETNEXT), 56(OPEN)
+| tuple-ids=27,20,13 row-size=495B cardinality=40.63M cost=27877122
+| in pipelines: 46(GETNEXT), 58(OPEN)
|
-|--F38:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=20 (adjusted from 120)
-| | Per-Instance Resources: mem-estimate=291.25MB mem-reservation=34.00MB thread-reservation=1
-| | max-parallelism=30 segment-costs=[56123197] cpu-comparison-result=60 [max(20 (self) vs 60 (sum children))]
+|--F41:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=106.89MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | max-parallelism=260 segment-costs=[76003415] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
| JOIN BUILD
| | join-table-id=09 plan-id=10 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.customer_0.c_customer_id
-| | runtime filters: RF007[min_max] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_id
-| | mem-estimate=274.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
-| |
-| 57:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_id)]
-| | mem-estimate=16.61MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=13 row-size=165B cardinality=30.00M cost=26123197
-| | in pipelines: 56(GETNEXT)
-| |
-| F13:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_id,tpcds_partitioned_parquet_snap.customer_0.c_first_name,tpcds_partitioned_parquet_snap.customer_0.c_last_name,tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_0.c_birth_country,tpcds_partitioned_parquet_snap.customer_0.c_login,tpcds_partitioned_parquet_snap.customer_0.c_email_address)] hosts=10 instances=40 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=190.62MB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=40 segment-costs=[226590531, 334518771] cpu-comparison-result=60 [max(40 (self) vs 60 (sum children))]
-| 56:AGGREGATE [FINALIZE]
+| | runtime filters: RF006[bloom] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF007[min_max] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_id
+| | mem-estimate=61.07MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=40626738
+| |
+| 59:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_id)]
+| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=13 row-size=165B cardinality=40.63M cost=35376677
+| | in pipelines: 58(GETNEXT)
+| |
+| F15:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_id,tpcds_partitioned_parquet_snap.customer_0.c_first_name,tpcds_partitioned_parquet_snap.customer_0.c_last_name,tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_0.c_birth_country,tpcds_partitioned_parquet_snap.customer_0.c_login,tpcds_partitioned_parquet_snap.customer_0.c_email_address)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=651.24MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=280 segment-costs=[2751800623, 453013550] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 58:AGGREGATE [FINALIZE]
| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales.ws_ext_sales_price), 2))
| | group by: tpcds_partitioned_parquet_snap.customer_0.c_customer_id, tpcds_partitioned_parquet_snap.customer_0.c_first_name, tpcds_partitioned_parquet_snap.customer_0.c_last_name, tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_0.c_birth_country, tpcds_partitioned_parquet_snap.customer_0.c_login, tpcds_partitioned_parquet_snap.customer_0.c_email_address
-| | mem-estimate=170.71MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=13 row-size=165B cardinality=30.00M cost=191394747
-| | in pipelines: 56(GETNEXT), 06(OPEN)
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales.ws_ext_sales_price), 2)) IS NOT NULL
+| | mem-estimate=571.96MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=13 row-size=165B cardinality=40.63M cost=2398033848
+| | in pipelines: 58(GETNEXT), 06(OPEN)
| |
-| 55:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_id,tpcds_partitioned_parquet_snap.customer_0.c_first_name,tpcds_partitioned_parquet_snap.customer_0.c_last_name,tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_0.c_birth_country,tpcds_partitioned_parquet_snap.customer_0.c_login,tpcds_partitioned_parquet_snap.customer_0.c_email_address)]
-| | mem-estimate=19.91MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=13 row-size=165B cardinality=40.42M cost=35195784
+| 57:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_id,tpcds_partitioned_parquet_snap.customer_0.c_first_name,tpcds_partitioned_parquet_snap.customer_0.c_last_name,tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_0.c_birth_country,tpcds_partitioned_parquet_snap.customer_0.c_login,tpcds_partitioned_parquet_snap.customer_0.c_email_address)]
+| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=13 row-size=165B cardinality=406.27M cost=353766775
| | in pipelines: 06(GETNEXT)
| |
-| F10:PLAN FRAGMENT [RANDOM] hosts=10 instances=60 (adjusted from 120)
-| Per-Host Shared Resources: mem-estimate=5.00MB mem-reservation=5.00MB thread-reservation=0 runtime-filters-memory=5.00MB
-| Per-Instance Resources: mem-estimate=128.27MB mem-reservation=34.12MB thread-reservation=1
-| max-parallelism=60 segment-costs=[580915243, 450697141] cpu-comparison-result=60 [max(60 (self) vs 31 (sum children))]
+| F14:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=522.94MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=460 segment-costs=[2715168496, 4530135523] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
| 11:AGGREGATE [STREAMING]
| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales.ws_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| | group by: tpcds_partitioned_parquet_snap.customer_0.c_customer_id, tpcds_partitioned_parquet_snap.customer_0.c_first_name, tpcds_partitioned_parquet_snap.customer_0.c_last_name, tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_0.c_birth_country, tpcds_partitioned_parquet_snap.customer_0.c_login, tpcds_partitioned_parquet_snap.customer_0.c_email_address
-| | mem-estimate=85.84MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=13 row-size=165B cardinality=40.42M cost=238891278
+| | mem-estimate=428.97MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=13 row-size=165B cardinality=406.27M cost=2398033848
| | in pipelines: 06(GETNEXT)
| |
-| 10:HASH JOIN [INNER JOIN, BROADCAST]
+| 10:HASH JOIN [INNER JOIN, PARTITIONED]
| | hash-table-id=10
| | hash predicates: tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=7,9,11 row-size=185B cardinality=40.65M cost=58349826
+| | tuple-ids=7,9,11 row-size=185B cardinality=406.27M cost=186854633
| | in pipelines: 06(GETNEXT), 09(OPEN)
| |
-| |--F39:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=549.69MB mem-reservation=208.00MB thread-reservation=1 runtime-filters-memory=4.00MB
-| | | max-parallelism=10 segment-costs=[6987000]
+| |--F42:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=54.45MB mem-reservation=34.00MB thread-reservation=1
+| | | max-parallelism=460 segment-costs=[54467197]
| | JOIN BUILD
| | | join-table-id=10 plan-id=11 cohort-id=05
| | | build expressions: tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
-| | | runtime filters: RF018[bloom] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_sk, RF019[min_max] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
-| | | mem-estimate=534.16MB mem-reservation=204.00MB spill-buffer=2.00MB thread-reservation=0 cost=3000000
+| | | runtime filters: RF019[min_max] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
| | |
-| | 54:EXCHANGE [BROADCAST]
+| | 56:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_sk)]
| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=11 row-size=153B cardinality=3.00M cost=3987000
+| | | tuple-ids=11 row-size=153B cardinality=30.00M cost=24467197
| | | in pipelines: 09(GETNEXT)
| | |
-| | F12:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Host Shared Resources: mem-estimate=4.00MB mem-reservation=4.00MB thread-reservation=0 runtime-filters-memory=4.00MB
-| | Per-Instance Resources: mem-estimate=16.61MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=10 segment-costs=[32019513]
+| | F13:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[342891999]
| | 09:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_0.c_customer_id)
-| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id
+| | predicates: tpcds_partitioned_parquet_snap.customer_0.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_0.c_customer_sk IS NOT NULL
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_0.c_customer_id)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_0.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_0.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=11 row-size=153B cardinality=3.00M cost=30714228
+| | tuple-ids=11 row-size=153B cardinality=30.00M cost=31557228
| | in pipelines: 09(GETNEXT)
| |
+| 55:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=7,9 row-size=32B cardinality=441.48M cost=130280015
+| | in pipelines: 06(GETNEXT)
+| |
+| F11:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=180 segment-costs=[1722951425]
| 08:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=11
| | hash predicates: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=7,9 row-size=32B cardinality=198.77M(filtered from 441.70M) cost=131021245
+| | tuple-ids=7,9 row-size=32B cardinality=441.48M cost=193234710
| | in pipelines: 06(GETNEXT), 07(OPEN)
| |
-| |--F40:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=12.64MB mem-reservation=12.62MB thread-reservation=1 runtime-filters-memory=1.00MB
+| |--F43:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
| | | join-table-id=11 plan-id=12 cohort-id=05
| | | build expressions: tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | | runtime filters: RF020[bloom] <- tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
-| | | mem-estimate=11.62MB mem-reservation=11.62MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
| | |
-| | 53:EXCHANGE [BROADCAST]
+| | 54:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
| | | tuple-ids=9 row-size=8B cardinality=373 cost=490
| | | in pipelines: 07(GETNEXT)
| | |
-| | F11:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | F12:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
+| | max-parallelism=1 segment-costs=[12536]
| | 07:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=9 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=9 row-size=8B cardinality=373 cost=12520
| | in pipelines: 07(GETNEXT)
| |
| 06:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
-| HDFS partitions=1824/1824 files=1824 size=145.75GB
-| runtime filters: RF019[min_max] -> tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk, RF018[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk, RF020[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
+| partition predicates: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=145.67GB
+| predicates: tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk IS NOT NULL
+| runtime filters: RF019[min_max] -> tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk, RF020[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
| stored statistics:
| table: rows=2.16G size=145.75GB
-| partitions: 1824/1824 rows=2.16G
+| partitions: 1823/1823 rows=2.16G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=374(filtered from 1824)
+| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=373(filtered from 1823)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=7 row-size=24B cardinality=198.77M(filtered from 2.16G) cost=152652894
+| tuple-ids=7 row-size=24B cardinality=441.48M(filtered from 2.16G) cost=165021409
| in pipelines: 06(GETNEXT)
|
24:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -1125,97 +1183,106 @@ max-parallelism=30 segment-costs=[286406984, 253] cpu-comparison-result=1020 [ma
| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_1.c_customer_id
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_1.c_customer_id
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20 row-size=330B cardinality=30.00M cost=47037753
-| in pipelines: 46(GETNEXT), 50(OPEN)
+| tuple-ids=27,20 row-size=330B cardinality=79.98M cost=52428030
+| in pipelines: 46(GETNEXT), 51(OPEN)
|
-|--F41:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=20 (adjusted from 120)
-| | Per-Instance Resources: mem-estimate=307.25MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
-| | max-parallelism=30 segment-costs=[56123197] cpu-comparison-result=120 [max(20 (self) vs 120 (sum children))]
+|--F44:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=169.95MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | max-parallelism=260 segment-costs=[149629463] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
| JOIN BUILD
| | join-table-id=12 plan-id=13 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.customer_1.c_customer_id
| | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF009[min_max] <- tpcds_partitioned_parquet_snap.customer_1.c_customer_id
-| | mem-estimate=274.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | mem-estimate=124.13MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=79982682
| |
-| 52:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_id)]
-| | mem-estimate=16.61MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=20 row-size=165B cardinality=30.00M cost=26123197
-| | in pipelines: 50(GETNEXT)
+| 53:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_id)]
+| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=20 row-size=165B cardinality=79.98M cost=69646781
+| | in pipelines: 51(GETNEXT)
| |
-| F08:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_id,tpcds_partitioned_parquet_snap.customer_1.c_first_name,tpcds_partitioned_parquet_snap.customer_1.c_last_name,tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_1.c_birth_country,tpcds_partitioned_parquet_snap.customer_1.c_login,tpcds_partitioned_parquet_snap.customer_1.c_email_address)] hosts=10 instances=40 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=367.23MB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=40 segment-costs=[315192267, 334518771] cpu-comparison-result=120 [max(40 (self) vs 120 (sum children))]
-| 50:AGGREGATE [FINALIZE]
+| F09:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_id,tpcds_partitioned_parquet_snap.customer_1.c_first_name,tpcds_partitioned_parquet_snap.customer_1.c_last_name,tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_1.c_birth_country,tpcds_partitioned_parquet_snap.customer_1.c_login,tpcds_partitioned_parquet_snap.customer_1.c_email_address)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.18GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=550 segment-costs=[5417525566, 891856952] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 51:AGGREGATE [FINALIZE]
| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_sales_price), 2))
| | group by: tpcds_partitioned_parquet_snap.customer_1.c_customer_id, tpcds_partitioned_parquet_snap.customer_1.c_first_name, tpcds_partitioned_parquet_snap.customer_1.c_last_name, tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_1.c_birth_country, tpcds_partitioned_parquet_snap.customer_1.c_login, tpcds_partitioned_parquet_snap.customer_1.c_email_address
-| | mem-estimate=337.40MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=20 row-size=165B cardinality=30.00M cost=245628386
-| | in pipelines: 50(GETNEXT), 12(OPEN)
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_sales_price), 2)) IS NOT NULL
+| | mem-estimate=1.10GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=20 row-size=165B cardinality=79.98M cost=4721057758
+| | in pipelines: 51(GETNEXT), 12(OPEN)
| |
-| 49:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_id,tpcds_partitioned_parquet_snap.customer_1.c_first_name,tpcds_partitioned_parquet_snap.customer_1.c_last_name,tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_1.c_birth_country,tpcds_partitioned_parquet_snap.customer_1.c_login,tpcds_partitioned_parquet_snap.customer_1.c_email_address)]
+| 50:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_id,tpcds_partitioned_parquet_snap.customer_1.c_first_name,tpcds_partitioned_parquet_snap.customer_1.c_last_name,tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_1.c_birth_country,tpcds_partitioned_parquet_snap.customer_1.c_login,tpcds_partitioned_parquet_snap.customer_1.c_email_address)]
| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=20 row-size=165B cardinality=79.89M cost=69563881
+| | tuple-ids=20 row-size=165B cardinality=799.83M cost=696467808
| | in pipelines: 12(GETNEXT)
| |
-| F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
-| Per-Host Shared Resources: mem-estimate=5.00MB mem-reservation=5.00MB thread-reservation=0 runtime-filters-memory=5.00MB
-| Per-Instance Resources: mem-estimate=127.73MB mem-reservation=35.00MB thread-reservation=1
-| max-parallelism=120 segment-costs=[1154475088, 890795390] cpu-comparison-result=120 [max(120 (self) vs 31 (sum children))]
+| F08:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=938.49MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=900 segment-costs=[5346593754, 8918569464] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
| 17:AGGREGATE [STREAMING]
| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| | group by: tpcds_partitioned_parquet_snap.customer_1.c_customer_id, tpcds_partitioned_parquet_snap.customer_1.c_first_name, tpcds_partitioned_parquet_snap.customer_1.c_last_name, tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_1.c_birth_country, tpcds_partitioned_parquet_snap.customer_1.c_login, tpcds_partitioned_parquet_snap.customer_1.c_email_address
-| | mem-estimate=85.30MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=20 row-size=165B cardinality=79.89M cost=472779967
+| | mem-estimate=844.52MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=20 row-size=165B cardinality=799.83M cost=4721057758
| | in pipelines: 12(GETNEXT)
| |
-| 16:HASH JOIN [INNER JOIN, BROADCAST]
+| 16:HASH JOIN [INNER JOIN, PARTITIONED]
| | hash-table-id=13
| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_1.c_customer_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_1.c_customer_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=14,16,18 row-size=185B cardinality=80.79M cost=116359176
+| | tuple-ids=14,16,18 row-size=185B cardinality=799.83M cost=368416277
| | in pipelines: 12(GETNEXT), 15(OPEN)
| |
-| |--F42:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=549.69MB mem-reservation=412.00MB thread-reservation=1 runtime-filters-memory=4.00MB
-| | | max-parallelism=10 segment-costs=[6987000]
+| |--F45:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=70.45MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | | max-parallelism=900 segment-costs=[54467197]
| | JOIN BUILD
| | | join-table-id=13 plan-id=14 cohort-id=06
| | | build expressions: tpcds_partitioned_parquet_snap.customer_1.c_customer_sk
| | | runtime filters: RF014[bloom] <- tpcds_partitioned_parquet_snap.customer_1.c_customer_sk, RF015[min_max] <- tpcds_partitioned_parquet_snap.customer_1.c_customer_sk
-| | | mem-estimate=534.16MB mem-reservation=408.00MB spill-buffer=2.00MB thread-reservation=0 cost=3000000
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
| | |
-| | 48:EXCHANGE [BROADCAST]
+| | 49:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_sk)]
| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=18 row-size=153B cardinality=3.00M cost=3987000
+| | | tuple-ids=18 row-size=153B cardinality=30.00M cost=24467197
| | | in pipelines: 15(GETNEXT)
| | |
| | F07:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Host Shared Resources: mem-estimate=4.00MB mem-reservation=4.00MB thread-reservation=0 runtime-filters-memory=4.00MB
-| | Per-Instance Resources: mem-estimate=16.61MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=10 segment-costs=[32019513]
+| | Per-Host Shared Resources: mem-estimate=16.00MB mem-reservation=16.00MB thread-reservation=0 runtime-filters-memory=16.00MB
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[342891999]
| | 15:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_1.c_customer_id)
-| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF007[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id
+| | predicates: tpcds_partitioned_parquet_snap.customer_1.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_1.c_customer_sk IS NOT NULL
+| | runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF006[bloom] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_1.c_customer_id)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_1.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_1.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=18 row-size=153B cardinality=3.00M cost=30714228
+| | tuple-ids=18 row-size=153B cardinality=30.00M cost=31557228
| | in pipelines: 15(GETNEXT)
| |
+| 48:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=14,16 row-size=32B cardinality=871.30M cost=257119719
+| | in pipelines: 12(GETNEXT)
+| |
+| F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=17.00MB mem-reservation=17.00MB thread-reservation=0 runtime-filters-memory=17.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=256.00KB thread-reservation=1
+| max-parallelism=350 segment-costs=[3401959876]
| 14:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=14
| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=14,16 row-size=32B cardinality=396.57M(filtered from 880.06M) cost=261187355
+| | tuple-ids=14,16 row-size=32B cardinality=871.30M cost=381366658
| | in pipelines: 12(GETNEXT), 13(OPEN)
| |
-| |--F43:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| |--F46:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
@@ -1231,61 +1298,64 @@ max-parallelism=30 segment-costs=[286406984, 253] cpu-comparison-result=1020 [ma
| | |
| | F06:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
+| | max-parallelism=1 segment-costs=[12536]
| | 13:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=16 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=16 row-size=8B cardinality=373 cost=12520
| | in pipelines: 13(GETNEXT)
| |
| 12:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
-| HDFS partitions=1831/1831 files=1831 size=280.96GB
-| runtime filters: RF015[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk, RF014[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk, RF016[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
+| partition predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk IS NOT NULL
+| HDFS partitions=1830/1831 files=1830 size=280.81GB
+| predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk IS NOT NULL
+| runtime filters: RF015[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk, RF016[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk, RF014[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk
| stored statistics:
| table: rows=4.32G size=280.96GB
-| partitions: 1831/1831 rows=4.32G
+| partitions: 1830/1830 rows=4.30G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=21.52M est-scan-range=374(filtered from 1831)
-| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=14 row-size=24B cardinality=396.57M(filtered from 4.32G) cost=304148590
+| extrapolated-rows=disabled max-scan-range-rows=5.09M est-scan-range=373(filtered from 1830)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=14 row-size=24B cardinality=871.30M(filtered from 4.28G) cost=327240201
| in pipelines: 12(GETNEXT)
|
-51:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)]
+52:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)]
| mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| tuple-ids=27 row-size=165B cardinality=162.19M cost=141230779
+| tuple-ids=27 row-size=165B cardinality=147.90M cost=128783355
| in pipelines: 46(GETNEXT)
|
F04:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id,tpcds_partitioned_parquet_snap.customer_2.c_first_name,tpcds_partitioned_parquet_snap.customer_2.c_last_name,tpcds_partitioned_parquet_snap.customer_2.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_2.c_birth_country,tpcds_partitioned_parquet_snap.customer_2.c_login,tpcds_partitioned_parquet_snap.customer_2.c_email_address)] hosts=10 instances=120
-Per-Instance Resources: mem-estimate=2.26GB mem-reservation=34.00MB thread-reservation=1
-max-parallelism=1100 segment-costs=[10985739011, 1808520801] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+Per-Instance Resources: mem-estimate=2.11GB mem-reservation=34.00MB thread-reservation=1
+max-parallelism=1010 segment-costs=[10017507045, 1649126185] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
46:AGGREGATE [FINALIZE]
| output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_sales_price), 2))
| group by: tpcds_partitioned_parquet_snap.customer_2.c_customer_id, tpcds_partitioned_parquet_snap.customer_2.c_first_name, tpcds_partitioned_parquet_snap.customer_2.c_last_name, tpcds_partitioned_parquet_snap.customer_2.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_2.c_birth_country, tpcds_partitioned_parquet_snap.customer_2.c_login, tpcds_partitioned_parquet_snap.customer_2.c_email_address
| having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
-| mem-estimate=2.23GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27 row-size=165B cardinality=162.19M cost=9573431219
+| mem-estimate=2.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=27 row-size=165B cardinality=147.90M cost=8729673496
| in pipelines: 46(GETNEXT), 18(OPEN)
|
45:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id,tpcds_partitioned_parquet_snap.customer_2.c_first_name,tpcds_partitioned_parquet_snap.customer_2.c_last_name,tpcds_partitioned_parquet_snap.customer_2.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_2.c_birth_country,tpcds_partitioned_parquet_snap.customer_2.c_login,tpcds_partitioned_parquet_snap.customer_2.c_email_address)]
| mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| tuple-ids=27 row-size=165B cardinality=1.62G cost=1412307792
+| tuple-ids=27 row-size=165B cardinality=1.48G cost=1287833549
| in pipelines: 18(GETNEXT)
|
F03:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk)] hosts=10 instances=120
-Per-Instance Resources: mem-estimate=1.76GB mem-reservation=34.00MB thread-reservation=1
-max-parallelism=1810 segment-costs=[10841902425, 18085207959] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+Per-Instance Resources: mem-estimate=1.62GB mem-reservation=34.00MB thread-reservation=1
+max-parallelism=1650 segment-costs=[9886347547, 16491261804] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
23:AGGREGATE [STREAMING]
| output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| group by: tpcds_partitioned_parquet_snap.customer_2.c_customer_id, tpcds_partitioned_parquet_snap.customer_2.c_first_name, tpcds_partitioned_parquet_snap.customer_2.c_last_name, tpcds_partitioned_parquet_snap.customer_2.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_2.c_birth_country, tpcds_partitioned_parquet_snap.customer_2.c_login, tpcds_partitioned_parquet_snap.customer_2.c_email_address
-| mem-estimate=1.67GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27 row-size=165B cardinality=1.62G cost=9573431219
+| mem-estimate=1.52GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=27 row-size=165B cardinality=1.48G cost=8729673496
| in pipelines: 18(GETNEXT)
|
22:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -1293,12 +1363,12 @@ max-parallelism=1810 segment-costs=[10841902425, 18085207959] cpu-comparison-res
| hash predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk = tpcds_partitioned_parquet_snap.customer_2.c_customer_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk = tpcds_partitioned_parquet_snap.customer_2.c_customer_sk
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=21,23,25 row-size=185B cardinality=1.62G cost=747080012
+| tuple-ids=21,23,25 row-size=185B cardinality=1.48G cost=681235853
| in pipelines: 18(GETNEXT), 21(OPEN)
|
-|--F44:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk)] hosts=10 instances=120
+|--F47:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk)] hosts=10 instances=120
| | Per-Instance Resources: mem-estimate=54.45MB mem-reservation=34.00MB thread-reservation=1
-| | max-parallelism=1810 segment-costs=[54467197]
+| | max-parallelism=1650 segment-costs=[54467197]
| JOIN BUILD
| | join-table-id=15 plan-id=16 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.customer_2.c_customer_sk
@@ -1311,38 +1381,40 @@ max-parallelism=1810 segment-costs=[10841902425, 18085207959] cpu-comparison-res
| | in pipelines: 21(GETNEXT)
| |
| F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| Per-Host Shared Resources: mem-estimate=20.00MB mem-reservation=20.00MB thread-reservation=0 runtime-filters-memory=20.00MB
+| Per-Host Shared Resources: mem-estimate=32.00MB mem-reservation=32.00MB thread-reservation=0 runtime-filters-memory=32.00MB
| Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
-| max-parallelism=10 segment-costs=[341205999]
+| max-parallelism=10 segment-costs=[342891999]
| 21:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
| HDFS partitions=1/1 files=1 size=1.55GB
-| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF009[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF007[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF008[bloom] -> tpcds [...]
+| predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_2.c_customer_sk IS NOT NULL
+| runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF009[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF006[bloom] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF008[bloom] -> tpcds [...]
| stored statistics:
| table: rows=30.00M size=1.55GB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=3.10M
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_2.c_customer_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=25 row-size=153B cardinality=30.00M cost=29871228
+| tuple-ids=25 row-size=153B cardinality=30.00M cost=31557228
| in pipelines: 21(GETNEXT)
|
43:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk)]
| mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
-| tuple-ids=21,23 row-size=32B cardinality=1.77G cost=521391194
+| tuple-ids=21,23 row-size=32B cardinality=1.61G cost=475438198
| in pipelines: 18(GETNEXT)
|
F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-Per-Instance Resources: mem-estimate=34.75MB mem-reservation=8.00MB thread-reservation=1
-max-parallelism=374 segment-costs=[6845578389]
+Per-Instance Resources: mem-estimate=34.75MB mem-reservation=512.00KB thread-reservation=1
+max-parallelism=373 segment-costs=[6315884965]
20:HASH JOIN [INNER JOIN, BROADCAST]
| hash-table-id=16
| hash predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_2.d_date_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales_0.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_2.d_date_sk
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=21,23 row-size=32B cardinality=1.77G cost=773340988
+| tuple-ids=21,23 row-size=32B cardinality=1.61G cost=705182309
| in pipelines: 18(GETNEXT), 19(OPEN)
|
-|--F45:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+|--F48:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
| | max-parallelism=10 segment-costs=[863]
| JOIN BUILD
@@ -1358,632 +1430,650 @@ max-parallelism=374 segment-costs=[6845578389]
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| max-parallelism=1 segment-costs=[10483]
+| max-parallelism=1 segment-costs=[12536]
| 19:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT)
+| predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_2.d_date_sk IS NOT NULL
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_2.d_date_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=23 row-size=8B cardinality=373 cost=10467
+| tuple-ids=23 row-size=8B cardinality=373 cost=12520
| in pipelines: 19(GETNEXT)
|
18:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk IS NOT NULL
runtime filters: RF011[min_max] -> tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk, RF012[bloom] -> tpcds_partitioned_parquet_snap.store_sales_0.ss_sold_date_sk
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=21 row-size=24B cardinality=1.77G(filtered from 8.64G) cost=610616051
+ extrapolated-rows=disabled max-scan-range-rows=12.22M est-scan-range=373(filtered from 1823)
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=21 row-size=24B cardinality=1.61G(filtered from 7.88G) cost=630443220
in pipelines: 18(GETNEXT)
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=6.02GB Threads=238
-Per-Host Resource Estimates: Memory=130.37GB
-F30:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Instance Resources: mem-estimate=4.17MB mem-reservation=4.00MB thread-reservation=1
-| max-parallelism=1 segment-costs=[442] cpu-comparison-result=1140 [max(1 (self) vs 1140 (sum children))]
+Max Per-Host Resource Reservation: Memory=10.17GB Threads=379
+Per-Host Resource Estimates: Memory=184.29GB
+F31:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Instance Resources: mem-estimate=5.02MB mem-reservation=4.00MB thread-reservation=1
+| max-parallelism=1 segment-costs=[442] cpu-comparison-result=1440 [max(1 (self) vs 1440 (sum children))]
PLAN-ROOT SINK
| output exprs: tpcds_partitioned_parquet_snap.customer.c_customer_id, tpcds_partitioned_parquet_snap.customer.c_first_name, tpcds_partitioned_parquet_snap.customer.c_last_name, tpcds_partitioned_parquet_snap.customer.c_birth_country
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=400
|
-77:MERGING-EXCHANGE [UNPARTITIONED]
+78:MERGING-EXCHANGE [UNPARTITIONED]
| order by: tpcds_partitioned_parquet_snap.customer.c_customer_id ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_first_name ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_last_name ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_birth_country ASC NULLS LAST
| limit: 100
-| mem-estimate=174.03KB mem-reservation=0B thread-reservation=0
+| mem-estimate=1.02MB mem-reservation=0B thread-reservation=0
| tuple-ids=42 row-size=85B cardinality=100 cost=42
| in pipelines: 41(GETNEXT)
|
-F09:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=20 (adjusted from 120)
+F10:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=120
Per-Instance Resources: mem-estimate=29.83MB mem-reservation=0B thread-reservation=1
-max-parallelism=30 segment-costs=[286406984, 253] cpu-comparison-result=1140 [max(20 (self) vs 1140 (sum children))]
+max-parallelism=260 segment-costs=[2591850306, 253] cpu-comparison-result=1440 [max(120 (self) vs 1440 (sum children))]
41:TOP-N [LIMIT=100]
| order by: tpcds_partitioned_parquet_snap.customer.c_customer_id ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_first_name ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_last_name ASC NULLS LAST, tpcds_partitioned_parquet_snap.customer.c_birth_country ASC NULLS LAST
| mem-estimate=8.27KB mem-reservation=0B thread-reservation=0
-| tuple-ids=42 row-size=85B cardinality=100 cost=50313918
+| tuple-ids=42 row-size=85B cardinality=100 cost=2263026260
| in pipelines: 41(GETNEXT), 46(OPEN)
|
40:HASH JOIN [INNER JOIN, PARTITIONED]
| hash-table-id=00
| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_4.c_customer_id
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_4.c_customer_id
-| other join predicates: CASE WHEN sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0)) THEN divide(sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_part [...]
+| other join predicates: CASE WHEN sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0)) THEN divide(sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_part [...]
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20,13,6,34,41 row-size=991B cardinality=4.06M cost=8431534
-| in pipelines: 46(GETNEXT), 75(OPEN)
+| tuple-ids=27,20,13,34,6,41 row-size=991B cardinality=147.90M cost=64733813
+| in pipelines: 46(GETNEXT), 76(OPEN)
|
-|--F31:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=20 (adjusted from 120)
-| | Per-Instance Resources: mem-estimate=55.76MB mem-reservation=38.00MB thread-reservation=1 runtime-filters-memory=4.00MB
-| | max-parallelism=30 segment-costs=[7604240] cpu-comparison-result=160 [max(20 (self) vs 160 (sum children))]
+|--F32:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=153.95MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=260 segment-costs=[149629463] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
| JOIN BUILD
| | join-table-id=00 plan-id=01 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.customer_4.c_customer_id
-| | runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_id, RF001[min_max] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_id
-| | mem-estimate=36.81MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=4064758
-| |
-| 76:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_id)]
-| | mem-estimate=14.96MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=41 row-size=165B cardinality=4.06M cost=3539482
-| | in pipelines: 75(GETNEXT)
-| |
-| F29:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_id,tpcds_partitioned_parquet_snap.customer_4.c_first_name,tpcds_partitioned_parquet_snap.customer_4.c_last_name,tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_4.c_birth_country,tpcds_partitioned_parquet_snap.customer_4.c_login,tpcds_partitioned_parquet_snap.customer_4.c_email_address)] hosts=10 instances=30 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=245.51MB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=30 segment-costs=[275321217, 45324595] cpu-comparison-result=160 [max(30 (self) vs 160 (sum children))]
-| 75:AGGREGATE [FINALIZE]
-| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), 2))
+| | runtime filters: RF001[min_max] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_id
+| | mem-estimate=124.13MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=79982682
+| |
+| 77:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_id)]
+| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=41 row-size=165B cardinality=79.98M cost=69646781
+| | in pipelines: 76(GETNEXT)
+| |
+| F30:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_id,tpcds_partitioned_parquet_snap.customer_4.c_first_name,tpcds_partitioned_parquet_snap.customer_4.c_last_name,tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_4.c_birth_country,tpcds_partitioned_parquet_snap.customer_4.c_login,tpcds_partitioned_parquet_snap.customer_4.c_email_address)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.18GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=550 segment-costs=[5417525566, 891856952] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 76:AGGREGATE [FINALIZE]
+| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2))
| | group by: tpcds_partitioned_parquet_snap.customer_4.c_customer_id, tpcds_partitioned_parquet_snap.customer_4.c_first_name, tpcds_partitioned_parquet_snap.customer_4.c_last_name, tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_4.c_birth_country, tpcds_partitioned_parquet_snap.customer_4.c_login, tpcds_partitioned_parquet_snap.customer_4.c_email_address
-| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
-| | mem-estimate=228.90MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=41 row-size=165B cardinality=4.06M cost=239926393
-| | in pipelines: 75(GETNEXT), 34(OPEN)
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
+| | mem-estimate=1.10GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=41 row-size=165B cardinality=79.98M cost=4721057758
+| | in pipelines: 76(GETNEXT), 34(OPEN)
| |
-| 74:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_id,tpcds_partitioned_parquet_snap.customer_4.c_first_name,tpcds_partitioned_parquet_snap.customer_4.c_last_name,tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_4.c_birth_country,tpcds_partitioned_parquet_snap.customer_4.c_login,tpcds_partitioned_parquet_snap.customer_4.c_email_address)]
-| | mem-estimate=16.61MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=41 row-size=165B cardinality=40.65M cost=35394824
+| 75:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_id,tpcds_partitioned_parquet_snap.customer_4.c_first_name,tpcds_partitioned_parquet_snap.customer_4.c_last_name,tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_4.c_birth_country,tpcds_partitioned_parquet_snap.customer_4.c_login,tpcds_partitioned_parquet_snap.customer_4.c_email_address)]
+| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=41 row-size=165B cardinality=799.83M cost=696467808
| | in pipelines: 34(GETNEXT)
| |
-| F28:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk)] hosts=10 instances=40 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=163.27MB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=50 segment-costs=[356933223, 453245929] cpu-comparison-result=160 [max(160 (self) vs 61 (sum children))]
+| F29:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=938.49MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=900 segment-costs=[5346593754, 8918569464] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
| 39:AGGREGATE [STREAMING]
-| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
+| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| | group by: tpcds_partitioned_parquet_snap.customer_4.c_customer_id, tpcds_partitioned_parquet_snap.customer_4.c_first_name, tpcds_partitioned_parquet_snap.customer_4.c_last_name, tpcds_partitioned_parquet_snap.customer_4.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_4.c_birth_country, tpcds_partitioned_parquet_snap.customer_4.c_login, tpcds_partitioned_parquet_snap.customer_4.c_email_address
-| | mem-estimate=128.76MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=41 row-size=165B cardinality=40.65M cost=239926393
+| | mem-estimate=844.52MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=41 row-size=165B cardinality=799.83M cost=4721057758
| | in pipelines: 34(GETNEXT)
| |
| 38:HASH JOIN [INNER JOIN, PARTITIONED]
| | hash-table-id=01
-| | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
-| | mem-estimate=0B mem-reservation=0B spill-buffer=1.00MB thread-reservation=0
-| | tuple-ids=35,37,39 row-size=185B cardinality=40.65M cost=58349826
+| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=35,37,39 row-size=185B cardinality=799.83M cost=368416277
| | in pipelines: 34(GETNEXT), 37(OPEN)
| |
-| |--F32:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk)] hosts=10 instances=40 (adjusted from 120)
-| | | Per-Instance Resources: mem-estimate=32.53MB mem-reservation=21.00MB thread-reservation=1 runtime-filters-memory=4.00MB
-| | | max-parallelism=50 segment-costs=[5446719]
+| |--F33:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=70.45MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | | max-parallelism=900 segment-costs=[54467197]
| | JOIN BUILD
| | | join-table-id=01 plan-id=02 cohort-id=02
| | | build expressions: tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
| | | runtime filters: RF030[bloom] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_sk, RF031[min_max] <- tpcds_partitioned_parquet_snap.customer_4.c_customer_sk
-| | | mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB thread-reservation=0 cost=3000000
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
| | |
-| | 73:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_sk)]
+| | 74:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_4.c_customer_sk)]
| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=39 row-size=153B cardinality=3.00M cost=2446719
+| | | tuple-ids=39 row-size=153B cardinality=30.00M cost=24467197
| | | in pipelines: 37(GETNEXT)
| | |
-| | F27:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Instance Resources: mem-estimate=40.55MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=10 segment-costs=[61847705]
+| | F28:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[342891999]
| | 37:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_4.c_customer_id)
+| | predicates: tpcds_partitioned_parquet_snap.customer_4.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_4.c_customer_sk IS NOT NULL
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_4.c_customer_id)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_4.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_4.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=39 row-size=153B cardinality=3.00M cost=30714228
+| | tuple-ids=39 row-size=153B cardinality=30.00M cost=31557228
| | in pipelines: 37(GETNEXT)
| |
-| 72:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk)]
+| 73:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk)]
| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=35,37 row-size=32B cardinality=198.77M(filtered from 441.70M) cost=58657004
+| | tuple-ids=35,37 row-size=32B cardinality=871.30M cost=257119719
| | in pipelines: 34(GETNEXT)
| |
-| F25:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
-| Per-Host Shared Resources: mem-estimate=4.00MB mem-reservation=4.00MB thread-reservation=0 runtime-filters-memory=4.00MB
-| Per-Instance Resources: mem-estimate=22.25MB mem-reservation=128.00KB thread-reservation=1
-| max-parallelism=150 segment-costs=[1491943763]
+| F26:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=17.00MB mem-reservation=17.00MB thread-reservation=0 runtime-filters-memory=17.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=256.00KB thread-reservation=1
+| max-parallelism=350 segment-costs=[3401959876]
| 36:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=02
-| | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=35,37 row-size=32B cardinality=198.77M(filtered from 441.70M) cost=131021245
+| | tuple-ids=35,37 row-size=32B cardinality=871.30M cost=381366658
| | in pipelines: 34(GETNEXT), 35(OPEN)
| |
-| |--F33:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=23.27MB mem-reservation=23.25MB thread-reservation=1
+| |--F34:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
| | | join-table-id=02 plan-id=03 cohort-id=02
| | | build expressions: tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
+| | | runtime filters: RF032[bloom] <- tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk
| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
| | |
-| | 71:EXCHANGE [BROADCAST]
+| | 72:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
| | | tuple-ids=37 row-size=8B cardinality=373 cost=490
| | | in pipelines: 35(GETNEXT)
| | |
-| | F26:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | F27:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
+| | max-parallelism=1 segment-costs=[12536]
| | 35:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_4.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_4.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=37 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=37 row-size=8B cardinality=373 cost=12520
| | in pipelines: 35(GETNEXT)
| |
-| 34:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
-| HDFS partitions=1824/1824 files=1824 size=145.75GB
-| runtime filters: RF031[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk, RF030[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk
+| 34:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
+| partition predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk IS NOT NULL
+| HDFS partitions=1830/1831 files=1830 size=280.81GB
+| predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk IS NOT NULL
+| runtime filters: RF031[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk, RF032[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk, RF030[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk
| stored statistics:
-| table: rows=2.16G size=145.75GB
-| partitions: 1824/1824 rows=2.16G
+| table: rows=4.32G size=280.96GB
+| partitions: 1830/1830 rows=4.30G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=3.37M
-| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=35 row-size=24B cardinality=198.77M(filtered from 2.16G) cost=746484929
+| extrapolated-rows=disabled max-scan-range-rows=5.09M est-scan-range=373(filtered from 1830)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=35 row-size=24B cardinality=871.30M(filtered from 4.28G) cost=327240201
| in pipelines: 34(GETNEXT)
|
33:HASH JOIN [INNER JOIN, PARTITIONED]
| hash-table-id=03
-| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_3.c_customer_id
-| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_3.c_customer_id
-| other join predicates: CASE WHEN sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0)) THEN divide(sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_part [...]
+| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer.c_customer_id
+| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer.c_customer_id
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20,13,6,34 row-size=826B cardinality=30.00M cost=13131000
-| in pipelines: 46(GETNEXT), 69(OPEN)
+| tuple-ids=27,20,13,34,6 row-size=826B cardinality=147.90M cost=37219403
+| in pipelines: 46(GETNEXT), 70(OPEN)
|
-|--F34:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=20 (adjusted from 120)
-| | Per-Instance Resources: mem-estimate=781.55MB mem-reservation=34.00MB thread-reservation=1
-| | max-parallelism=30 segment-costs=[151134290] cpu-comparison-result=240 [max(20 (self) vs 240 (sum children))]
+|--F35:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=264.52MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=260 segment-costs=[276678749] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
| JOIN BUILD
| | join-table-id=03 plan-id=04 cohort-id=01
-| | build expressions: tpcds_partitioned_parquet_snap.customer_3.c_customer_id
-| | runtime filters: RF003[min_max] <- tpcds_partitioned_parquet_snap.customer_3.c_customer_id
-| | mem-estimate=751.73MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=80787070
+| | build expressions: tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | runtime filters: RF003[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | mem-estimate=234.70MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=147895394
| |
-| 70:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_id)]
+| 71:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_id)]
| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=34 row-size=165B cardinality=80.79M cost=70347220
-| | in pipelines: 69(GETNEXT)
+| | tuple-ids=6 row-size=165B cardinality=147.90M cost=128783355
+| | in pipelines: 70(GETNEXT)
| |
-| F24:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_id,tpcds_partitioned_parquet_snap.customer_3.c_first_name,tpcds_partitioned_parquet_snap.customer_3.c_last_name,tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_3.c_birth_country,tpcds_partitioned_parquet_snap.customer_3.c_login,tpcds_partitioned_parquet_snap.customer_3.c_email_address)] hosts=10 instances=120
-| Per-Instance Resources: mem-estimate=1.14GB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=550 segment-costs=[5472009822, 900826381] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
-| 69:AGGREGATE [FINALIZE]
-| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2))
-| | group by: tpcds_partitioned_parquet_snap.customer_3.c_customer_id, tpcds_partitioned_parquet_snap.customer_3.c_first_name, tpcds_partitioned_parquet_snap.customer_3.c_last_name, tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_3.c_birth_country, tpcds_partitioned_parquet_snap.customer_3.c_login, tpcds_partitioned_parquet_snap.customer_3.c_email_address
-| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
-| | mem-estimate=1.11GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=34 row-size=165B cardinality=80.79M cost=4768537611
-| | in pipelines: 69(GETNEXT), 27(OPEN)
+| F25:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_id,tpcds_partitioned_parquet_snap.customer.c_first_name,tpcds_partitioned_parquet_snap.customer.c_last_name,tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer.c_birth_country,tpcds_partitioned_parquet_snap.customer.c_login,tpcds_partitioned_parquet_snap.customer.c_email_address)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=2.11GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=1010 segment-costs=[10017507045, 1649126185] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 70:AGGREGATE [FINALIZE]
+| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales.ss_ext_sales_price), 2))
+| | group by: tpcds_partitioned_parquet_snap.customer.c_customer_id, tpcds_partitioned_parquet_snap.customer.c_first_name, tpcds_partitioned_parquet_snap.customer.c_last_name, tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer.c_birth_country, tpcds_partitioned_parquet_snap.customer.c_login, tpcds_partitioned_parquet_snap.customer.c_email_address
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales.ss_ext_sales_price), 2)) IS NOT NULL
+| | mem-estimate=2.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=6 row-size=165B cardinality=147.90M cost=8729673496
+| | in pipelines: 70(GETNEXT), 00(OPEN)
| |
-| 68:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_id,tpcds_partitioned_parquet_snap.customer_3.c_first_name,tpcds_partitioned_parquet_snap.customer_3.c_last_name,tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_3.c_birth_country,tpcds_partitioned_parquet_snap.customer_3.c_login,tpcds_partitioned_parquet_snap.customer_3.c_email_address)]
+| 69:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_id,tpcds_partitioned_parquet_snap.customer.c_first_name,tpcds_partitioned_parquet_snap.customer.c_last_name,tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer.c_birth_country,tpcds_partitioned_parquet_snap.customer.c_login,tpcds_partitioned_parquet_snap.customer.c_email_address)]
| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=34 row-size=165B cardinality=807.87M cost=703472211
-| | in pipelines: 27(GETNEXT)
+| | tuple-ids=6 row-size=165B cardinality=1.48G cost=1287833549
+| | in pipelines: 00(GETNEXT)
| |
-| F23:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk)] hosts=10 instances=120
-| Per-Instance Resources: mem-estimate=946.99MB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=910 segment-costs=[5400364645, 9008263849] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
-| 32:AGGREGATE [STREAMING]
-| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales_0.cs_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
-| | group by: tpcds_partitioned_parquet_snap.customer_3.c_customer_id, tpcds_partitioned_parquet_snap.customer_3.c_first_name, tpcds_partitioned_parquet_snap.customer_3.c_last_name, tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_3.c_birth_country, tpcds_partitioned_parquet_snap.customer_3.c_login, tpcds_partitioned_parquet_snap.customer_3.c_email_address
-| | mem-estimate=853.01MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=34 row-size=165B cardinality=807.87M cost=4768537611
-| | in pipelines: 27(GETNEXT)
+| F24:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.62GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=1650 segment-costs=[9886347547, 16491261804] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 05:AGGREGATE [STREAMING]
+| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales.ss_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
+| | group by: tpcds_partitioned_parquet_snap.customer.c_customer_id, tpcds_partitioned_parquet_snap.customer.c_first_name, tpcds_partitioned_parquet_snap.customer.c_last_name, tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer.c_birth_country, tpcds_partitioned_parquet_snap.customer.c_login, tpcds_partitioned_parquet_snap.customer.c_email_address
+| | mem-estimate=1.52GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=6 row-size=165B cardinality=1.48G cost=8729673496
+| | in pipelines: 00(GETNEXT)
| |
-| 31:HASH JOIN [INNER JOIN, PARTITIONED]
+| 04:HASH JOIN [INNER JOIN, PARTITIONED]
| | hash-table-id=04
-| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=28,30,32 row-size=185B cardinality=807.87M cost=372121453
-| | in pipelines: 27(GETNEXT), 30(OPEN)
+| | tuple-ids=0,2,4 row-size=185B cardinality=1.48G cost=681235853
+| | in pipelines: 00(GETNEXT), 03(OPEN)
| |
-| |--F35:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk)] hosts=10 instances=120
+| |--F36:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=120
| | | Per-Instance Resources: mem-estimate=54.45MB mem-reservation=34.00MB thread-reservation=1
-| | | max-parallelism=910 segment-costs=[54467197]
+| | | max-parallelism=1650 segment-costs=[54467197]
| | JOIN BUILD
| | | join-table-id=04 plan-id=05 cohort-id=03
-| | | build expressions: tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
-| | | runtime filters: RF027[min_max] <- tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
+| | | build expressions: tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | | runtime filters: RF027[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk
| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
| | |
-| | 67:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_sk)]
+| | 68:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_sk)]
| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=32 row-size=153B cardinality=30.00M cost=24467197
-| | | in pipelines: 30(GETNEXT)
+| | | tuple-ids=4 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 03(GETNEXT)
| | |
-| | F22:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Host Shared Resources: mem-estimate=4.00MB mem-reservation=4.00MB thread-reservation=0 runtime-filters-memory=4.00MB
+| | F23:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=10 segment-costs=[341205999]
-| | 30:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | max-parallelism=10 segment-costs=[342891999]
+| | 03:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_3.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer_3.c_customer_id
+| | predicates: tpcds_partitioned_parquet_snap.customer.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=32 row-size=153B cardinality=30.00M cost=29871228
-| | in pipelines: 30(GETNEXT)
+| | tuple-ids=4 row-size=153B cardinality=30.00M cost=31557228
+| | in pipelines: 03(GETNEXT)
| |
-| 66:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk)]
+| 67:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)]
| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=28,30 row-size=32B cardinality=880.06M cost=259705581
-| | in pipelines: 27(GETNEXT)
+| | tuple-ids=0,2 row-size=32B cardinality=1.61G cost=475438198
+| | in pipelines: 00(GETNEXT)
| |
-| F20:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| F21:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=1.00MB thread-reservation=1
-| max-parallelism=350 segment-costs=[3409790834]
-| 29:HASH JOIN [INNER JOIN, BROADCAST]
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=512.00KB thread-reservation=1
+| max-parallelism=373 segment-costs=[6315884965]
+| 02:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=05
-| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=28,30 row-size=32B cardinality=880.06M cost=385202077
-| | in pipelines: 27(GETNEXT), 28(OPEN)
+| | tuple-ids=0,2 row-size=32B cardinality=1.61G cost=705182309
+| | in pipelines: 00(GETNEXT), 01(OPEN)
| |
-| |--F36:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| |--F37:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
| | | join-table-id=05 plan-id=06 cohort-id=03
-| | | build expressions: tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
-| | | runtime filters: RF028[bloom] <- tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
+| | | build expressions: tpcds_partitioned_parquet_snap.date_dim.d_date_sk
+| | | runtime filters: RF028[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
| | |
-| | 65:EXCHANGE [BROADCAST]
+| | 66:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=30 row-size=8B cardinality=373 cost=490
-| | | in pipelines: 28(GETNEXT)
+| | | tuple-ids=2 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 01(GETNEXT)
| | |
-| | F21:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | F22:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
-| | 28:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | max-parallelism=1 segment-costs=[12536]
+| | 01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
-| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT)
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=30 row-size=8B cardinality=373 cost=10467
-| | in pipelines: 28(GETNEXT)
+| | tuple-ids=2 row-size=8B cardinality=373 cost=12520
+| | in pipelines: 01(GETNEXT)
| |
-| 27:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
-| HDFS partitions=1831/1831 files=1831 size=280.96GB
-| runtime filters: RF027[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_bill_customer_sk, RF028[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales_0.cs_sold_date_sk
+| 00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
+| partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=389.69GB
+| predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk IS NOT NULL
+| runtime filters: RF027[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF028[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk
| stored statistics:
-| table: rows=4.32G size=280.96GB
-| partitions: 1831/1831 rows=4.32G
+| table: rows=8.64G size=389.90GB
+| partitions: 1823/1823 rows=8.25G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=21.52M est-scan-range=374(filtered from 1831)
-| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=28 row-size=24B cardinality=880.06M(filtered from 4.32G) cost=304148590
-| in pipelines: 27(GETNEXT)
+| extrapolated-rows=disabled max-scan-range-rows=12.22M est-scan-range=373(filtered from 1823)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+| tuple-ids=0 row-size=24B cardinality=1.61G(filtered from 7.88G) cost=630443220
+| in pipelines: 00(GETNEXT)
|
-26:HASH JOIN [INNER JOIN, PARTITIONED]
+32:HASH JOIN [INNER JOIN, PARTITIONED]
| hash-table-id=06
-| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer.c_customer_id
-| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer.c_customer_id
+| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_3.c_customer_id
+| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_3.c_customer_id
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20,13,6 row-size=661B cardinality=30.00M cost=13131000
-| in pipelines: 46(GETNEXT), 63(OPEN)
+| tuple-ids=27,20,13,34 row-size=661B cardinality=40.63M cost=17782323
+| in pipelines: 46(GETNEXT), 64(OPEN)
|
-|--F37:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=20 (adjusted from 120)
-| | Per-Instance Resources: mem-estimate=304.47MB mem-reservation=34.00MB thread-reservation=1
-| | max-parallelism=30 segment-costs=[56123197] cpu-comparison-result=240 [max(20 (self) vs 240 (sum children))]
+|--F38:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=90.89MB mem-reservation=34.00MB thread-reservation=1
+| | max-parallelism=260 segment-costs=[76003415] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
| JOIN BUILD
| | join-table-id=06 plan-id=07 cohort-id=01
-| | build expressions: tpcds_partitioned_parquet_snap.customer.c_customer_id
-| | runtime filters: RF005[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_id
-| | mem-estimate=274.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | build expressions: tpcds_partitioned_parquet_snap.customer_3.c_customer_id
+| | runtime filters: RF005[min_max] <- tpcds_partitioned_parquet_snap.customer_3.c_customer_id
+| | mem-estimate=61.07MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=40626738
| |
-| 64:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_id)]
+| 65:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_id)]
| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=6 row-size=165B cardinality=30.00M cost=26123197
-| | in pipelines: 63(GETNEXT)
+| | tuple-ids=34 row-size=165B cardinality=40.63M cost=35376677
+| | in pipelines: 64(GETNEXT)
| |
-| F19:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_id,tpcds_partitioned_parquet_snap.customer.c_first_name,tpcds_partitioned_parquet_snap.customer.c_last_name,tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer.c_birth_country,tpcds_partitioned_parquet_snap.customer.c_login,tpcds_partitioned_parquet_snap.customer.c_email_address)] hosts=10 instances=120
-| Per-Instance Resources: mem-estimate=1.82GB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=310 segment-costs=[3067099075, 334518771] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
-| 63:AGGREGATE [FINALIZE]
-| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales.ss_ext_sales_price), 2))
-| | group by: tpcds_partitioned_parquet_snap.customer.c_customer_id, tpcds_partitioned_parquet_snap.customer.c_first_name, tpcds_partitioned_parquet_snap.customer.c_last_name, tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer.c_birth_country, tpcds_partitioned_parquet_snap.customer.c_login, tpcds_partitioned_parquet_snap.customer.c_email_address
-| | mem-estimate=1.80GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=6 row-size=165B cardinality=30.00M cost=1930086569
-| | in pipelines: 63(GETNEXT), 00(OPEN)
+| F20:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_id,tpcds_partitioned_parquet_snap.customer_3.c_first_name,tpcds_partitioned_parquet_snap.customer_3.c_last_name,tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_3.c_birth_country,tpcds_partitioned_parquet_snap.customer_3.c_login,tpcds_partitioned_parquet_snap.customer_3.c_email_address)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=651.24MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=280 segment-costs=[2751800623, 453013550] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 64:AGGREGATE [FINALIZE]
+| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), 2))
+| | group by: tpcds_partitioned_parquet_snap.customer_3.c_customer_id, tpcds_partitioned_parquet_snap.customer_3.c_first_name, tpcds_partitioned_parquet_snap.customer_3.c_last_name, tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_3.c_birth_country, tpcds_partitioned_parquet_snap.customer_3.c_login, tpcds_partitioned_parquet_snap.customer_3.c_email_address
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
+| | mem-estimate=571.96MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=34 row-size=165B cardinality=40.63M cost=2398033848
+| | in pipelines: 64(GETNEXT), 26(OPEN)
| |
-| 62:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_id,tpcds_partitioned_parquet_snap.customer.c_first_name,tpcds_partitioned_parquet_snap.customer.c_last_name,tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer.c_birth_country,tpcds_partitioned_parquet_snap.customer.c_login,tpcds_partitioned_parquet_snap.customer.c_email_address)]
+| 63:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_id,tpcds_partitioned_parquet_snap.customer_3.c_first_name,tpcds_partitioned_parquet_snap.customer_3.c_last_name,tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_3.c_birth_country,tpcds_partitioned_parquet_snap.customer_3.c_login,tpcds_partitioned_parquet_snap.customer_3.c_email_address)]
| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=6 row-size=165B cardinality=1.31G cost=1137012506
-| | in pipelines: 00(GETNEXT)
+| | tuple-ids=34 row-size=165B cardinality=406.27M cost=353766775
+| | in pipelines: 26(GETNEXT)
| |
-| F18:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=120
-| Per-Instance Resources: mem-estimate=1.76GB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=1460 segment-costs=[9410215497, 14559933564] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
-| 05:AGGREGATE [STREAMING]
-| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales.ss_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
-| | group by: tpcds_partitioned_parquet_snap.customer.c_customer_id, tpcds_partitioned_parquet_snap.customer.c_first_name, tpcds_partitioned_parquet_snap.customer.c_last_name, tpcds_partitioned_parquet_snap.customer.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer.c_birth_country, tpcds_partitioned_parquet_snap.customer.c_login, tpcds_partitioned_parquet_snap.customer.c_email_address
-| | mem-estimate=1.67GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=6 row-size=165B cardinality=1.31G cost=8141744291
-| | in pipelines: 00(GETNEXT)
+| F19:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=522.94MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=460 segment-costs=[2715168496, 4530135523] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+| 31:AGGREGATE [STREAMING]
+| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales_0.ws_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
+| | group by: tpcds_partitioned_parquet_snap.customer_3.c_customer_id, tpcds_partitioned_parquet_snap.customer_3.c_first_name, tpcds_partitioned_parquet_snap.customer_3.c_last_name, tpcds_partitioned_parquet_snap.customer_3.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_3.c_birth_country, tpcds_partitioned_parquet_snap.customer_3.c_login, tpcds_partitioned_parquet_snap.customer_3.c_email_address
+| | mem-estimate=428.97MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=34 row-size=165B cardinality=406.27M cost=2398033848
+| | in pipelines: 26(GETNEXT)
| |
-| 04:HASH JOIN [INNER JOIN, PARTITIONED]
+| 30:HASH JOIN [INNER JOIN, PARTITIONED]
| | hash-table-id=07
-| | hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=0,2,4 row-size=185B cardinality=1.62G cost=747080012
-| | in pipelines: 00(GETNEXT), 03(OPEN)
+| | tuple-ids=28,30,32 row-size=185B cardinality=406.27M cost=186854633
+| | in pipelines: 26(GETNEXT), 29(OPEN)
| |
-| |--F38:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=120
+| |--F39:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk)] hosts=10 instances=120
| | | Per-Instance Resources: mem-estimate=54.45MB mem-reservation=34.00MB thread-reservation=1
-| | | max-parallelism=1460 segment-costs=[54467197]
+| | | max-parallelism=460 segment-costs=[54467197]
| | JOIN BUILD
| | | join-table-id=07 plan-id=08 cohort-id=04
-| | | build expressions: tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| | | runtime filters: RF023[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | | build expressions: tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
+| | | runtime filters: RF023[min_max] <- tpcds_partitioned_parquet_snap.customer_3.c_customer_sk
| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
| | |
-| | 61:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_sk)]
+| | 62:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_3.c_customer_sk)]
| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=4 row-size=153B cardinality=30.00M cost=24467197
-| | | in pipelines: 03(GETNEXT)
+| | | tuple-ids=32 row-size=153B cardinality=30.00M cost=24467197
+| | | in pipelines: 29(GETNEXT)
| | |
-| | F17:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Host Shared Resources: mem-estimate=4.00MB mem-reservation=4.00MB thread-reservation=0 runtime-filters-memory=4.00MB
+| | F18:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=10 segment-costs=[341205999]
-| | 03:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | max-parallelism=10 segment-costs=[342891999]
+| | 29:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer.c_customer_id
+| | predicates: tpcds_partitioned_parquet_snap.customer_3.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_3.c_customer_sk IS NOT NULL
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_3.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_3.c_customer_id
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_3.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_3.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=4 row-size=153B cardinality=30.00M cost=29871228
-| | in pipelines: 03(GETNEXT)
+| | tuple-ids=32 row-size=153B cardinality=30.00M cost=31557228
+| | in pipelines: 29(GETNEXT)
| |
-| 60:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)]
+| 61:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk)]
| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=0,2 row-size=32B cardinality=1.77G cost=521391194
-| | in pipelines: 00(GETNEXT)
+| | tuple-ids=28,30 row-size=32B cardinality=441.48M cost=130280015
+| | in pipelines: 26(GETNEXT)
| |
-| F15:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| F16:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=8.00MB thread-reservation=1
-| max-parallelism=374 segment-costs=[6845578389]
-| 02:HASH JOIN [INNER JOIN, BROADCAST]
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=180 segment-costs=[1722951425]
+| 28:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=08
-| | hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
+| | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=0,2 row-size=32B cardinality=1.77G cost=773340988
-| | in pipelines: 00(GETNEXT), 01(OPEN)
+| | tuple-ids=28,30 row-size=32B cardinality=441.48M cost=193234710
+| | in pipelines: 26(GETNEXT), 27(OPEN)
| |
-| |--F39:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| |--F40:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
| | | join-table-id=08 plan-id=09 cohort-id=04
-| | | build expressions: tpcds_partitioned_parquet_snap.date_dim.d_date_sk
-| | | runtime filters: RF024[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
+| | | build expressions: tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
+| | | runtime filters: RF024[bloom] <- tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk
| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
| | |
-| | 59:EXCHANGE [BROADCAST]
+| | 60:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=2 row-size=8B cardinality=373 cost=490
-| | | in pipelines: 01(GETNEXT)
+| | | tuple-ids=30 row-size=8B cardinality=373 cost=490
+| | | in pipelines: 27(GETNEXT)
| | |
-| | F16:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | F17:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
-| | 01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
+| | max-parallelism=1 segment-costs=[12536]
+| | 27:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
-| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_3.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_3.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=2 row-size=8B cardinality=373 cost=10467
-| | in pipelines: 01(GETNEXT)
-| |
-| 00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
-| HDFS partitions=1824/1824 files=1824 size=389.90GB
-| runtime filters: RF023[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF024[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk
+| | tuple-ids=30 row-size=8B cardinality=373 cost=12520
+| | in pipelines: 27(GETNEXT)
+| |
+| 26:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
+| partition predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=145.67GB
+| predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk IS NOT NULL
+| runtime filters: RF023[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk, RF024[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_sold_date_sk
| stored statistics:
-| table: rows=8.64G size=389.90GB
-| partitions: 1824/1824 rows=8.64G
+| table: rows=2.16G size=145.75GB
+| partitions: 1823/1823 rows=2.16G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
-| mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
-| tuple-ids=0 row-size=24B cardinality=1.77G(filtered from 8.64G) cost=610616051
-| in pipelines: 00(GETNEXT)
+| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=373(filtered from 1823)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_bill_customer_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=28 row-size=24B cardinality=441.48M(filtered from 2.16G) cost=165021409
+| in pipelines: 26(GETNEXT)
|
25:HASH JOIN [INNER JOIN, PARTITIONED]
| hash-table-id=09
| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_0.c_customer_id
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_0.c_customer_id
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20,13 row-size=495B cardinality=30.00M cost=13131000
-| in pipelines: 46(GETNEXT), 57(OPEN)
+| tuple-ids=27,20,13 row-size=495B cardinality=40.63M cost=27877122
+| in pipelines: 46(GETNEXT), 58(OPEN)
|
-|--F40:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=20 (adjusted from 120)
-| | Per-Instance Resources: mem-estimate=291.25MB mem-reservation=34.00MB thread-reservation=1
-| | max-parallelism=30 segment-costs=[56123197] cpu-comparison-result=140 [max(20 (self) vs 140 (sum children))]
+|--F41:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=106.89MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | max-parallelism=260 segment-costs=[76003415] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
| JOIN BUILD
| | join-table-id=09 plan-id=10 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.customer_0.c_customer_id
-| | runtime filters: RF007[min_max] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_id
-| | mem-estimate=274.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
-| |
-| 58:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_id)]
-| | mem-estimate=16.61MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=13 row-size=165B cardinality=30.00M cost=26123197
-| | in pipelines: 57(GETNEXT)
-| |
-| F14:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_id,tpcds_partitioned_parquet_snap.customer_0.c_first_name,tpcds_partitioned_parquet_snap.customer_0.c_last_name,tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_0.c_birth_country,tpcds_partitioned_parquet_snap.customer_0.c_login,tpcds_partitioned_parquet_snap.customer_0.c_email_address)] hosts=10 instances=40 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=188.97MB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=40 segment-costs=[226590531, 334518771] cpu-comparison-result=140 [max(40 (self) vs 140 (sum children))]
-| 57:AGGREGATE [FINALIZE]
+| | runtime filters: RF006[bloom] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF007[min_max] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_id
+| | mem-estimate=61.07MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=40626738
+| |
+| 59:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_id)]
+| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=13 row-size=165B cardinality=40.63M cost=35376677
+| | in pipelines: 58(GETNEXT)
+| |
+| F15:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_id,tpcds_partitioned_parquet_snap.customer_0.c_first_name,tpcds_partitioned_parquet_snap.customer_0.c_last_name,tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_0.c_birth_country,tpcds_partitioned_parquet_snap.customer_0.c_login,tpcds_partitioned_parquet_snap.customer_0.c_email_address)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=651.24MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=280 segment-costs=[2751800623, 453013550] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 58:AGGREGATE [FINALIZE]
| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales.ws_ext_sales_price), 2))
| | group by: tpcds_partitioned_parquet_snap.customer_0.c_customer_id, tpcds_partitioned_parquet_snap.customer_0.c_first_name, tpcds_partitioned_parquet_snap.customer_0.c_last_name, tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_0.c_birth_country, tpcds_partitioned_parquet_snap.customer_0.c_login, tpcds_partitioned_parquet_snap.customer_0.c_email_address
-| | mem-estimate=170.71MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=13 row-size=165B cardinality=30.00M cost=191394747
-| | in pipelines: 57(GETNEXT), 06(OPEN)
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales.ws_ext_sales_price), 2)) IS NOT NULL
+| | mem-estimate=571.96MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=13 row-size=165B cardinality=40.63M cost=2398033848
+| | in pipelines: 58(GETNEXT), 06(OPEN)
| |
-| 56:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_id,tpcds_partitioned_parquet_snap.customer_0.c_first_name,tpcds_partitioned_parquet_snap.customer_0.c_last_name,tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_0.c_birth_country,tpcds_partitioned_parquet_snap.customer_0.c_login,tpcds_partitioned_parquet_snap.customer_0.c_email_address)]
-| | mem-estimate=18.26MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=13 row-size=165B cardinality=40.42M cost=35195784
+| 57:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_id,tpcds_partitioned_parquet_snap.customer_0.c_first_name,tpcds_partitioned_parquet_snap.customer_0.c_last_name,tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_0.c_birth_country,tpcds_partitioned_parquet_snap.customer_0.c_login,tpcds_partitioned_parquet_snap.customer_0.c_email_address)]
+| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=13 row-size=165B cardinality=406.27M cost=353766775
| | in pipelines: 06(GETNEXT)
| |
-| F13:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk)] hosts=10 instances=50 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=142.95MB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=50 segment-costs=[355898108, 450697141] cpu-comparison-result=140 [max(140 (self) vs 71 (sum children))]
+| F14:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=522.94MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=460 segment-costs=[2715168496, 4530135523] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
| 11:AGGREGATE [STREAMING]
| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.web_sales.ws_ext_list_price, tpcds_partitioned_parquet_snap.web_sales.ws_ext_wholesale_cost), tpcds_partitioned_parquet_snap.web_sales.ws_ext_discount_amt), tpcds_partitioned_parquet_snap.web_sales.ws_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| | group by: tpcds_partitioned_parquet_snap.customer_0.c_customer_id, tpcds_partitioned_parquet_snap.customer_0.c_first_name, tpcds_partitioned_parquet_snap.customer_0.c_last_name, tpcds_partitioned_parquet_snap.customer_0.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_0.c_birth_country, tpcds_partitioned_parquet_snap.customer_0.c_login, tpcds_partitioned_parquet_snap.customer_0.c_email_address
-| | mem-estimate=103.01MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=13 row-size=165B cardinality=40.42M cost=238891278
+| | mem-estimate=428.97MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=13 row-size=165B cardinality=406.27M cost=2398033848
| | in pipelines: 06(GETNEXT)
| |
| 10:HASH JOIN [INNER JOIN, PARTITIONED]
| | hash-table-id=10
| | hash predicates: tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
-| | mem-estimate=0B mem-reservation=0B spill-buffer=1.00MB thread-reservation=0
-| | tuple-ids=7,9,11 row-size=185B cardinality=40.65M cost=58349826
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=7,9,11 row-size=185B cardinality=406.27M cost=186854633
| | in pipelines: 06(GETNEXT), 09(OPEN)
| |
-| |--F41:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk)] hosts=10 instances=50 (adjusted from 120)
-| | | Per-Instance Resources: mem-estimate=32.53MB mem-reservation=21.00MB thread-reservation=1 runtime-filters-memory=4.00MB
-| | | max-parallelism=50 segment-costs=[5446719]
+| |--F42:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=54.45MB mem-reservation=34.00MB thread-reservation=1
+| | | max-parallelism=460 segment-costs=[54467197]
| | JOIN BUILD
| | | join-table-id=10 plan-id=11 cohort-id=05
| | | build expressions: tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
-| | | runtime filters: RF018[bloom] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_sk, RF019[min_max] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
-| | | mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB thread-reservation=0 cost=3000000
+| | | runtime filters: RF019[min_max] <- tpcds_partitioned_parquet_snap.customer_0.c_customer_sk
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
| | |
-| | 55:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_sk)]
+| | 56:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_0.c_customer_sk)]
| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=11 row-size=153B cardinality=3.00M cost=2446719
+| | | tuple-ids=11 row-size=153B cardinality=30.00M cost=24467197
| | | in pipelines: 09(GETNEXT)
| | |
-| | F12:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Host Shared Resources: mem-estimate=4.00MB mem-reservation=4.00MB thread-reservation=0 runtime-filters-memory=4.00MB
-| | Per-Instance Resources: mem-estimate=46.69MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=10 segment-costs=[61847705]
+| | F13:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[342891999]
| | 09:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_0.c_customer_id)
-| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id
+| | predicates: tpcds_partitioned_parquet_snap.customer_0.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_0.c_customer_sk IS NOT NULL
+| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_0.c_customer_id
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_0.c_customer_id)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_0.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_0.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=11 row-size=153B cardinality=3.00M cost=30714228
+| | tuple-ids=11 row-size=153B cardinality=30.00M cost=31557228
| | in pipelines: 09(GETNEXT)
| |
-| 54:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk)]
-| | mem-estimate=13.52MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=7,9 row-size=32B cardinality=198.77M(filtered from 441.70M) cost=58657004
+| 55:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=7,9 row-size=32B cardinality=441.48M cost=130280015
| | in pipelines: 06(GETNEXT)
| |
-| F10:PLAN FRAGMENT [RANDOM] hosts=10 instances=90 (adjusted from 120)
-| Per-Host Shared Resources: mem-estimate=5.00MB mem-reservation=5.00MB thread-reservation=0 runtime-filters-memory=5.00MB
-| Per-Instance Resources: mem-estimate=23.81MB mem-reservation=128.00KB thread-reservation=1
-| max-parallelism=90 segment-costs=[898111728]
+| F11:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=180 segment-costs=[1722951425]
| 08:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=11
| | hash predicates: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=7,9 row-size=32B cardinality=198.77M(filtered from 441.70M) cost=131021245
+| | tuple-ids=7,9 row-size=32B cardinality=441.48M cost=193234710
| | in pipelines: 06(GETNEXT), 07(OPEN)
| |
-| |--F42:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=18.45MB mem-reservation=18.44MB thread-reservation=1 runtime-filters-memory=1.00MB
+| |--F43:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
| | | join-table-id=11 plan-id=12 cohort-id=05
| | | build expressions: tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | | runtime filters: RF020[bloom] <- tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
-| | | mem-estimate=17.44MB mem-reservation=17.44MB spill-buffer=64.00KB thread-reservation=0 cost=373
+| | | mem-estimate=23.25MB mem-reservation=23.25MB spill-buffer=64.00KB thread-reservation=0 cost=373
| | |
-| | 53:EXCHANGE [BROADCAST]
+| | 54:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
| | | tuple-ids=9 row-size=8B cardinality=373 cost=490
| | | in pipelines: 07(GETNEXT)
| | |
-| | F11:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
+| | F12:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
+| | max-parallelism=1 segment-costs=[12536]
| | 07:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=9 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=9 row-size=8B cardinality=373 cost=12520
| | in pipelines: 07(GETNEXT)
| |
| 06:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
-| HDFS partitions=1824/1824 files=1824 size=145.75GB
-| runtime filters: RF019[min_max] -> tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk, RF018[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk, RF020[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
+| partition predicates: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=145.67GB
+| predicates: tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk IS NOT NULL
+| runtime filters: RF019[min_max] -> tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk, RF020[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
| stored statistics:
| table: rows=2.16G size=145.75GB
-| partitions: 1824/1824 rows=2.16G
+| partitions: 1823/1823 rows=2.16G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=374(filtered from 1824)
+| extrapolated-rows=disabled max-scan-range-rows=3.37M est-scan-range=373(filtered from 1823)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_sales.ws_bill_customer_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=7 row-size=24B cardinality=198.77M(filtered from 2.16G) cost=152652894
+| tuple-ids=7 row-size=24B cardinality=441.48M(filtered from 2.16G) cost=165021409
| in pipelines: 06(GETNEXT)
|
24:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -1991,97 +2081,106 @@ max-parallelism=30 segment-costs=[286406984, 253] cpu-comparison-result=1140 [ma
| hash predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_1.c_customer_id
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_2.c_customer_id = tpcds_partitioned_parquet_snap.customer_1.c_customer_id
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27,20 row-size=330B cardinality=30.00M cost=47037753
-| in pipelines: 46(GETNEXT), 50(OPEN)
+| tuple-ids=27,20 row-size=330B cardinality=79.98M cost=52428030
+| in pipelines: 46(GETNEXT), 51(OPEN)
|
-|--F43:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=20 (adjusted from 120)
-| | Per-Instance Resources: mem-estimate=307.25MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
-| | max-parallelism=30 segment-costs=[56123197] cpu-comparison-result=120 [max(20 (self) vs 120 (sum children))]
+|--F44:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)] hosts=10 instances=120
+| | Per-Instance Resources: mem-estimate=169.95MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | max-parallelism=260 segment-costs=[149629463] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
| JOIN BUILD
| | join-table-id=12 plan-id=13 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.customer_1.c_customer_id
| | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF009[min_max] <- tpcds_partitioned_parquet_snap.customer_1.c_customer_id
-| | mem-estimate=274.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | mem-estimate=124.13MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=79982682
| |
-| 52:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_id)]
-| | mem-estimate=16.61MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=20 row-size=165B cardinality=30.00M cost=26123197
-| | in pipelines: 50(GETNEXT)
+| 53:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_id)]
+| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=20 row-size=165B cardinality=79.98M cost=69646781
+| | in pipelines: 51(GETNEXT)
| |
-| F08:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_id,tpcds_partitioned_parquet_snap.customer_1.c_first_name,tpcds_partitioned_parquet_snap.customer_1.c_last_name,tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_1.c_birth_country,tpcds_partitioned_parquet_snap.customer_1.c_login,tpcds_partitioned_parquet_snap.customer_1.c_email_address)] hosts=10 instances=40 (adjusted from 120)
-| Per-Instance Resources: mem-estimate=367.23MB mem-reservation=34.00MB thread-reservation=1
-| max-parallelism=40 segment-costs=[315192267, 334518771] cpu-comparison-result=120 [max(40 (self) vs 120 (sum children))]
-| 50:AGGREGATE [FINALIZE]
+| F09:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_id,tpcds_partitioned_parquet_snap.customer_1.c_first_name,tpcds_partitioned_parquet_snap.customer_1.c_last_name,tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_1.c_birth_country,tpcds_partitioned_parquet_snap.customer_1.c_login,tpcds_partitioned_parquet_snap.customer_1.c_email_address)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=1.18GB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=550 segment-costs=[5417525566, 891856952] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+| 51:AGGREGATE [FINALIZE]
| | output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_sales_price), 2))
| | group by: tpcds_partitioned_parquet_snap.customer_1.c_customer_id, tpcds_partitioned_parquet_snap.customer_1.c_first_name, tpcds_partitioned_parquet_snap.customer_1.c_last_name, tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_1.c_birth_country, tpcds_partitioned_parquet_snap.customer_1.c_login, tpcds_partitioned_parquet_snap.customer_1.c_email_address
-| | mem-estimate=337.40MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=20 row-size=165B cardinality=30.00M cost=245628386
-| | in pipelines: 50(GETNEXT), 12(OPEN)
+| | having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_sales_price), 2)) IS NOT NULL
+| | mem-estimate=1.10GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=20 row-size=165B cardinality=79.98M cost=4721057758
+| | in pipelines: 51(GETNEXT), 12(OPEN)
| |
-| 49:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_id,tpcds_partitioned_parquet_snap.customer_1.c_first_name,tpcds_partitioned_parquet_snap.customer_1.c_last_name,tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_1.c_birth_country,tpcds_partitioned_parquet_snap.customer_1.c_login,tpcds_partitioned_parquet_snap.customer_1.c_email_address)]
+| 50:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_id,tpcds_partitioned_parquet_snap.customer_1.c_first_name,tpcds_partitioned_parquet_snap.customer_1.c_last_name,tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_1.c_birth_country,tpcds_partitioned_parquet_snap.customer_1.c_login,tpcds_partitioned_parquet_snap.customer_1.c_email_address)]
| | mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=20 row-size=165B cardinality=79.89M cost=69563881
+| | tuple-ids=20 row-size=165B cardinality=799.83M cost=696467808
| | in pipelines: 12(GETNEXT)
| |
-| F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
-| Per-Host Shared Resources: mem-estimate=5.00MB mem-reservation=5.00MB thread-reservation=0 runtime-filters-memory=5.00MB
-| Per-Instance Resources: mem-estimate=127.73MB mem-reservation=35.00MB thread-reservation=1
-| max-parallelism=120 segment-costs=[1154475088, 890795390] cpu-comparison-result=120 [max(120 (self) vs 31 (sum children))]
+| F08:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk)] hosts=10 instances=120
+| Per-Instance Resources: mem-estimate=938.49MB mem-reservation=34.00MB thread-reservation=1
+| max-parallelism=900 segment-costs=[5346593754, 8918569464] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
| 17:AGGREGATE [STREAMING]
| | output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_list_price, tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_wholesale_cost), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_discount_amt), tpcds_partitioned_parquet_snap.catalog_sales.cs_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| | group by: tpcds_partitioned_parquet_snap.customer_1.c_customer_id, tpcds_partitioned_parquet_snap.customer_1.c_first_name, tpcds_partitioned_parquet_snap.customer_1.c_last_name, tpcds_partitioned_parquet_snap.customer_1.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_1.c_birth_country, tpcds_partitioned_parquet_snap.customer_1.c_login, tpcds_partitioned_parquet_snap.customer_1.c_email_address
-| | mem-estimate=85.30MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=20 row-size=165B cardinality=79.89M cost=472779967
+| | mem-estimate=844.52MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=20 row-size=165B cardinality=799.83M cost=4721057758
| | in pipelines: 12(GETNEXT)
| |
-| 16:HASH JOIN [INNER JOIN, BROADCAST]
+| 16:HASH JOIN [INNER JOIN, PARTITIONED]
| | hash-table-id=13
| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_1.c_customer_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk = tpcds_partitioned_parquet_snap.customer_1.c_customer_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=14,16,18 row-size=185B cardinality=80.79M cost=116359176
+| | tuple-ids=14,16,18 row-size=185B cardinality=799.83M cost=368416277
| | in pipelines: 12(GETNEXT), 15(OPEN)
| |
-| |--F44:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=549.69MB mem-reservation=412.00MB thread-reservation=1 runtime-filters-memory=4.00MB
-| | | max-parallelism=10 segment-costs=[6987000]
+| |--F45:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk)] hosts=10 instances=120
+| | | Per-Instance Resources: mem-estimate=70.45MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | | max-parallelism=900 segment-costs=[54467197]
| | JOIN BUILD
| | | join-table-id=13 plan-id=14 cohort-id=06
| | | build expressions: tpcds_partitioned_parquet_snap.customer_1.c_customer_sk
| | | runtime filters: RF014[bloom] <- tpcds_partitioned_parquet_snap.customer_1.c_customer_sk, RF015[min_max] <- tpcds_partitioned_parquet_snap.customer_1.c_customer_sk
-| | | mem-estimate=534.16MB mem-reservation=408.00MB spill-buffer=2.00MB thread-reservation=0 cost=3000000
+| | | mem-estimate=42.91MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
| | |
-| | 48:EXCHANGE [BROADCAST]
+| | 49:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_1.c_customer_sk)]
| | | mem-estimate=11.53MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=18 row-size=153B cardinality=3.00M cost=3987000
+| | | tuple-ids=18 row-size=153B cardinality=30.00M cost=24467197
| | | in pipelines: 15(GETNEXT)
| | |
| | F07:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Host Shared Resources: mem-estimate=4.00MB mem-reservation=4.00MB thread-reservation=0 runtime-filters-memory=4.00MB
-| | Per-Instance Resources: mem-estimate=16.61MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=10 segment-costs=[32019513]
+| | Per-Host Shared Resources: mem-estimate=16.00MB mem-reservation=16.00MB thread-reservation=0 runtime-filters-memory=16.00MB
+| | Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[342891999]
| | 15:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
| | HDFS partitions=1/1 files=1 size=1.55GB
-| | predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_1.c_customer_id)
-| | runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF007[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id
+| | predicates: tpcds_partitioned_parquet_snap.customer_1.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_1.c_customer_sk IS NOT NULL
+| | runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id, RF006[bloom] -> tpcds_partitioned_parquet_snap.customer_1.c_customer_id
| | stored statistics:
| | table: rows=30.00M size=1.55GB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.customer_1.c_customer_id)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_1.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_1.c_customer_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=18 row-size=153B cardinality=3.00M cost=30714228
+| | tuple-ids=18 row-size=153B cardinality=30.00M cost=31557228
| | in pipelines: 15(GETNEXT)
| |
+| 48:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk)]
+| | mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=14,16 row-size=32B cardinality=871.30M cost=257119719
+| | in pipelines: 12(GETNEXT)
+| |
+| F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+| Per-Host Shared Resources: mem-estimate=17.00MB mem-reservation=17.00MB thread-reservation=0 runtime-filters-memory=17.00MB
+| Per-Instance Resources: mem-estimate=34.75MB mem-reservation=256.00KB thread-reservation=1
+| max-parallelism=350 segment-costs=[3401959876]
| 14:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=14
| | hash predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk
| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=14,16 row-size=32B cardinality=396.57M(filtered from 880.06M) cost=261187355
+| | tuple-ids=14,16 row-size=32B cardinality=871.30M cost=381366658
| | in pipelines: 12(GETNEXT), 13(OPEN)
| |
-| |--F45:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| |--F46:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=10 segment-costs=[863]
| | JOIN BUILD
@@ -2097,61 +2196,64 @@ max-parallelism=30 segment-costs=[286406984, 253] cpu-comparison-result=1140 [ma
| | |
| | F06:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[10483]
+| | max-parallelism=1 segment-costs=[12536]
| | 13:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT))
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_year = add(CAST(1999 AS INT), CAST(1 AS INT)), tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=16 row-size=8B cardinality=373 cost=10467
+| | tuple-ids=16 row-size=8B cardinality=373 cost=12520
| | in pipelines: 13(GETNEXT)
| |
| 12:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
-| HDFS partitions=1831/1831 files=1831 size=280.96GB
-| runtime filters: RF015[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk, RF014[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk, RF016[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
+| partition predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk IS NOT NULL
+| HDFS partitions=1830/1831 files=1830 size=280.81GB
+| predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk IS NOT NULL
+| runtime filters: RF015[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk, RF016[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk, RF014[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk
| stored statistics:
| table: rows=4.32G size=280.96GB
-| partitions: 1831/1831 rows=4.32G
+| partitions: 1830/1830 rows=4.30G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=21.52M est-scan-range=374(filtered from 1831)
-| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=14 row-size=24B cardinality=396.57M(filtered from 4.32G) cost=304148590
+| extrapolated-rows=disabled max-scan-range-rows=5.09M est-scan-range=373(filtered from 1830)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_bill_customer_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=14 row-size=24B cardinality=871.30M(filtered from 4.28G) cost=327240201
| in pipelines: 12(GETNEXT)
|
-51:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)]
+52:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id)]
| mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| tuple-ids=27 row-size=165B cardinality=162.19M cost=141230779
+| tuple-ids=27 row-size=165B cardinality=147.90M cost=128783355
| in pipelines: 46(GETNEXT)
|
F04:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id,tpcds_partitioned_parquet_snap.customer_2.c_first_name,tpcds_partitioned_parquet_snap.customer_2.c_last_name,tpcds_partitioned_parquet_snap.customer_2.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_2.c_birth_country,tpcds_partitioned_parquet_snap.customer_2.c_login,tpcds_partitioned_parquet_snap.customer_2.c_email_address)] hosts=10 instances=120
-Per-Instance Resources: mem-estimate=2.26GB mem-reservation=34.00MB thread-reservation=1
-max-parallelism=1100 segment-costs=[10985739011, 1808520801] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
+Per-Instance Resources: mem-estimate=2.11GB mem-reservation=34.00MB thread-reservation=1
+max-parallelism=1010 segment-costs=[10017507045, 1649126185] cpu-comparison-result=240 [max(120 (self) vs 240 (sum children))]
46:AGGREGATE [FINALIZE]
| output: sum:merge(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_sales_price), 2))
| group by: tpcds_partitioned_parquet_snap.customer_2.c_customer_id, tpcds_partitioned_parquet_snap.customer_2.c_first_name, tpcds_partitioned_parquet_snap.customer_2.c_last_name, tpcds_partitioned_parquet_snap.customer_2.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_2.c_birth_country, tpcds_partitioned_parquet_snap.customer_2.c_login, tpcds_partitioned_parquet_snap.customer_2.c_email_address
| having: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_sales_price), 2)) > CAST(0 AS DECIMAL(3,0))
-| mem-estimate=2.23GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27 row-size=165B cardinality=162.19M cost=9573431219
+| mem-estimate=2.03GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=27 row-size=165B cardinality=147.90M cost=8729673496
| in pipelines: 46(GETNEXT), 18(OPEN)
|
45:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_2.c_customer_id,tpcds_partitioned_parquet_snap.customer_2.c_first_name,tpcds_partitioned_parquet_snap.customer_2.c_last_name,tpcds_partitioned_parquet_snap.customer_2.c_preferred_cust_flag,tpcds_partitioned_parquet_snap.customer_2.c_birth_country,tpcds_partitioned_parquet_snap.customer_2.c_login,tpcds_partitioned_parquet_snap.customer_2.c_email_address)]
| mem-estimate=29.82MB mem-reservation=0B thread-reservation=0
-| tuple-ids=27 row-size=165B cardinality=1.62G cost=1412307792
+| tuple-ids=27 row-size=165B cardinality=1.48G cost=1287833549
| in pipelines: 18(GETNEXT)
|
F03:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk)] hosts=10 instances=120
-Per-Instance Resources: mem-estimate=1.76GB mem-reservation=34.00MB thread-reservation=1
-max-parallelism=1810 segment-costs=[10841902425, 18085207959] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
+Per-Instance Resources: mem-estimate=1.62GB mem-reservation=34.00MB thread-reservation=1
+max-parallelism=1650 segment-costs=[9886347547, 16491261804] cpu-comparison-result=240 [max(240 (self) vs 141 (sum children))]
23:AGGREGATE [STREAMING]
| output: sum(divide(add(subtract(subtract(tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_list_price, tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_wholesale_cost), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_discount_amt), tpcds_partitioned_parquet_snap.store_sales_0.ss_ext_sales_price), CAST(2 AS DECIMAL(3,0))))
| group by: tpcds_partitioned_parquet_snap.customer_2.c_customer_id, tpcds_partitioned_parquet_snap.customer_2.c_first_name, tpcds_partitioned_parquet_snap.customer_2.c_last_name, tpcds_partitioned_parquet_snap.customer_2.c_preferred_cust_flag, tpcds_partitioned_parquet_snap.customer_2.c_birth_country, tpcds_partitioned_parquet_snap.customer_2.c_login, tpcds_partitioned_parquet_snap.customer_2.c_email_address
-| mem-estimate=1.67GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=27 row-size=165B cardinality=1.62G cost=9573431219
+| mem-estimate=1.52GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=27 row-size=165B cardinality=1.48G cost=8729673496
| in pipelines: 18(GETNEXT)
|
22:HASH JOIN [INNER JOIN, PARTITIONED]
@@ -2159,12 +2261,12 @@ max-parallelism=1810 segment-costs=[10841902425, 18085207959] cpu-comparison-res
| hash predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk = tpcds_partitioned_parquet_snap.customer_2.c_customer_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk = tpcds_partitioned_parquet_snap.customer_2.c_customer_sk
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=21,23,25 row-size=185B cardinality=1.62G cost=747080012
+| tuple-ids=21,23,25 row-size=185B cardinality=1.48G cost=681235853
| in pipelines: 18(GETNEXT), 21(OPEN)
|
-|--F46:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk)] hosts=10 instances=120
+|--F47:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk)] hosts=10 instances=120
| | Per-Instance Resources: mem-estimate=54.45MB mem-reservation=34.00MB thread-reservation=1
-| | max-parallelism=1810 segment-costs=[54467197]
+| | max-parallelism=1650 segment-costs=[54467197]
| JOIN BUILD
| | join-table-id=15 plan-id=16 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.customer_2.c_customer_sk
@@ -2177,38 +2279,40 @@ max-parallelism=1810 segment-costs=[10841902425, 18085207959] cpu-comparison-res
| | in pipelines: 21(GETNEXT)
| |
| F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| Per-Host Shared Resources: mem-estimate=20.00MB mem-reservation=20.00MB thread-reservation=0 runtime-filters-memory=20.00MB
+| Per-Host Shared Resources: mem-estimate=32.00MB mem-reservation=32.00MB thread-reservation=0 runtime-filters-memory=32.00MB
| Per-Instance Resources: mem-estimate=89.66MB mem-reservation=1.00MB thread-reservation=1
-| max-parallelism=10 segment-costs=[341205999]
+| max-parallelism=10 segment-costs=[342891999]
| 21:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
| HDFS partitions=1/1 files=1 size=1.55GB
-| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF009[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF007[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF000[bloom] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF008[bloom] -> tpcds [...]
+| predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_2.c_customer_sk IS NOT NULL
+| runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF009[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF001[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF005[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF003[min_max] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF006[bloom] -> tpcds_partitioned_parquet_snap.customer_2.c_customer_id, RF008[bloom] -> tpcds [...]
| stored statistics:
| table: rows=30.00M size=1.55GB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=3.10M
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_2.c_customer_id IS NOT NULL, tpcds_partitioned_parquet_snap.customer_2.c_customer_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=25 row-size=153B cardinality=30.00M cost=29871228
+| tuple-ids=25 row-size=153B cardinality=30.00M cost=31557228
| in pipelines: 21(GETNEXT)
|
43:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk)]
| mem-estimate=14.69MB mem-reservation=0B thread-reservation=0
-| tuple-ids=21,23 row-size=32B cardinality=1.77G cost=521391194
+| tuple-ids=21,23 row-size=32B cardinality=1.61G cost=475438198
| in pipelines: 18(GETNEXT)
|
F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-Per-Instance Resources: mem-estimate=34.75MB mem-reservation=8.00MB thread-reservation=1
-max-parallelism=374 segment-costs=[6845578389]
+Per-Instance Resources: mem-estimate=34.75MB mem-reservation=512.00KB thread-reservation=1
+max-parallelism=373 segment-costs=[6315884965]
20:HASH JOIN [INNER JOIN, BROADCAST]
| hash-table-id=16
| hash predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_2.d_date_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales_0.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim_2.d_date_sk
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=21,23 row-size=32B cardinality=1.77G cost=773340988
+| tuple-ids=21,23 row-size=32B cardinality=1.61G cost=705182309
| in pipelines: 18(GETNEXT), 19(OPEN)
|
-|--F47:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+|--F48:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
| | Per-Instance Resources: mem-estimate=24.27MB mem-reservation=24.25MB thread-reservation=1 runtime-filters-memory=1.00MB
| | max-parallelism=10 segment-costs=[863]
| JOIN BUILD
@@ -2224,29 +2328,32 @@ max-parallelism=374 segment-costs=[6845578389]
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=16.05MB mem-reservation=512.00KB thread-reservation=1
-| max-parallelism=1 segment-costs=[10483]
+| max-parallelism=1 segment-costs=[12536]
| 19:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT)
+| predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_2.d_date_sk IS NOT NULL
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_2.d_year = CAST(1999 AS INT), tpcds_partitioned_parquet_snap.date_dim_2.d_date_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=23 row-size=8B cardinality=373 cost=10467
+| tuple-ids=23 row-size=8B cardinality=373 cost=12520
| in pipelines: 19(GETNEXT)
|
18:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk IS NOT NULL
runtime filters: RF011[min_max] -> tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk, RF012[bloom] -> tpcds_partitioned_parquet_snap.store_sales_0.ss_sold_date_sk
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=21 row-size=24B cardinality=1.77G(filtered from 8.64G) cost=610616051
+ extrapolated-rows=disabled max-scan-range-rows=12.22M est-scan-range=373(filtered from 1823)
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales_0.ss_customer_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=21 row-size=24B cardinality=1.61G(filtered from 7.88G) cost=630443220
in pipelines: 18(GETNEXT)
====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q05.test b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q05.test
index afde2702f..54290be99 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q05.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q05.test
@@ -128,11 +128,11 @@ with ssr as
# end query 5 in stream 0 using template query5.tpl
---- PLAN
-Max Per-Host Resource Reservation: Memory=133.12MB Threads=1
-Per-Host Resource Estimates: Memory=11.18GB
+Max Per-Host Resource Reservation: Memory=132.91MB Threads=1
+Per-Host Resource Estimates: Memory=10.97GB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Instance Resources: mem-estimate=11.18GB mem-reservation=133.12MB thread-reservation=1 runtime-filters-memory=23.00MB
-| max-parallelism=1 segment-costs=[16252493382, 8126622616, 7582950483, 508568, 443665, 669120, 500]
+| Per-Instance Resources: mem-estimate=10.97GB mem-reservation=132.91MB thread-reservation=1 runtime-filters-memory=23.00MB
+| max-parallelism=1 segment-costs=[15196560009, 8178168907, 7578166738, 508568, 443665, 669120, 500]
PLAN-ROOT SINK
| output exprs: CASE valid_tid(34,35,36) WHEN 34 THEN CHANNEL WHEN 35 THEN CHANNEL WHEN 36 THEN NULL END, CASE valid_tid(34,35,36) WHEN 34 THEN ID WHEN 35 THEN NULL WHEN 36 THEN NULL END, aggif(valid_tid(34,35,36) IN (34, 35, 36), CASE valid_tid(34,35,36) WHEN 34 THEN sum(SALES) WHEN 35 THEN sum(SALES) WHEN 36 THEN sum(SALES) END), aggif(valid_tid(34,35,36) IN (34, 35, 36), CASE valid_tid(34,35,36) WHEN 34 THEN sum(returns) WHEN 35 THEN sum(returns) WHEN 36 THEN sum(returns) END), aggif [...]
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=500
@@ -174,7 +174,7 @@ PLAN-ROOT SINK
| | output: sum(SALES_PRICE), sum(PROFIT), sum(RETURN_AMT), sum(NET_LOSS)
| | group by: tpcds_partitioned_parquet_snap.web_site.web_site_id
| | mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=32 row-size=92B cardinality=33 cost=1855940925
+| | tuple-ids=32 row-size=92B cardinality=33 cost=1821623700
| | in pipelines: 27(GETNEXT), 19(OPEN), 20(OPEN)
| |
| 26:HASH JOIN [INNER JOIN]
@@ -182,17 +182,19 @@ PLAN-ROOT SINK
| | fk/pk conjuncts: assumed fk/pk
| | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.web_site.web_site_sk, RF009[min_max] <- tpcds_partitioned_parquet_snap.web_site.web_site_sk
| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=27,28,30 row-size=64B cardinality=3.69G cost=1616607638
+| | tuple-ids=27,28,30 row-size=64B cardinality=3.63G cost=1586715799
| | in pipelines: 19(GETNEXT), 20(GETNEXT), 25(OPEN)
| |
| |--25:SCAN HDFS [tpcds_partitioned_parquet_snap.web_site]
| | HDFS partitions=1/1 files=1 size=17.88KB
+| | predicates: tpcds_partitioned_parquet_snap.web_site.web_site_sk IS NOT NULL
| | stored statistics:
| | table: rows=66 size=17.88KB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=66
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_site.web_site_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
-| | tuple-ids=30 row-size=32B cardinality=66 cost=19
+| | tuple-ids=30 row-size=32B cardinality=66 cost=20
| | in pipelines: 25(GETNEXT)
| |
| 24:HASH JOIN [INNER JOIN]
@@ -200,69 +202,75 @@ PLAN-ROOT SINK
| | fk/pk conjuncts: assumed fk/pk
| | runtime filters: RF010[bloom] <- tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk
| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=27,28 row-size=32B cardinality=3.69G cost=1616614877
+| | tuple-ids=27,28 row-size=32B cardinality=3.63G cost=1586723038
| | in pipelines: 19(GETNEXT), 20(GETNEXT), 23(OPEN)
| |
| |--23:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=28 row-size=8B cardinality=7.30K cost=12520
+| | tuple-ids=28 row-size=8B cardinality=7.30K cost=14573
| | in pipelines: 23(GETNEXT)
| |
| 18:UNION
| | mem-estimate=0B mem-reservation=0B thread-reservation=0
-| | tuple-ids=27 row-size=24B cardinality=3.69G cost=375915738
+| | tuple-ids=27 row-size=24B cardinality=3.63G cost=368964878
| | in pipelines: 19(GETNEXT), 20(GETNEXT)
| |
-| |--22:HASH JOIN [RIGHT OUTER JOIN]
+| |--22:HASH JOIN [INNER JOIN]
| | | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk = tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number = tpcds_partitioned_parquet_snap.web_returns.wr_order_number
| | | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk = tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number = tpcds_partitioned_parquet_snap.web_returns.wr_order_number
| | | runtime filters: RF012[bloom] <- tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, RF013[bloom] <- tpcds_partitioned_parquet_snap.web_returns.wr_order_number, RF014[min_max] <- tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, RF015[min_max] <- tpcds_partitioned_parquet_snap.web_returns.wr_order_number
-| | | mem-estimate=10.83GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | | tuple-ids=23N,25 row-size=40B cardinality=1.53G cost=1047895000
+| | | mem-estimate=10.62GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | | tuple-ids=23,25 row-size=40B cardinality=1.47G cost=1026170294
| | | in pipelines: 20(GETNEXT), 21(OPEN)
| | |
| | |--21:SCAN HDFS [tpcds_partitioned_parquet_snap.web_returns]
-| | | HDFS partitions=2114/2114 files=2114 size=16.74GB
+| | | partition predicates: tpcds_partitioned_parquet_snap.web_returns.wr_returned_date_sk IS NOT NULL
+| | | HDFS partitions=2113/2114 files=2113 size=16.73GB
| | | runtime filters: RF010[bloom] -> tpcds_partitioned_parquet_snap.web_returns.wr_returned_date_sk
| | | stored statistics:
| | | table: rows=216.00M size=16.74GB
-| | | partitions: 2114/2114 rows=216.00M
+| | | partitions: 2113/2113 rows=206.59M
| | | columns: all
-| | | extrapolated-rows=disabled max-scan-range-rows=9.41M
-| | | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
-| | | tuple-ids=25 row-size=24B cardinality=216.00M cost=74650528
+| | | extrapolated-rows=disabled max-scan-range-rows=198.66K
+| | | mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+| | | tuple-ids=25 row-size=24B cardinality=206.59M cost=71397691
| | | in pipelines: 21(GETNEXT)
| | |
| | 20:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
| | HDFS partitions=1824/1824 files=1824 size=145.75GB
+| | predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_web_site_sk IS NOT NULL
| | runtime filters: RF014[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk, RF015[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number, RF009[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_web_site_sk, RF012[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk, RF013[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number, RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_web_site_sk
| | stored statistics:
| | table: rows=2.16G size=145.75GB
| | partitions: 1824/1824 rows=2.16G
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.37M
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_web_site_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| | tuple-ids=23 row-size=16B cardinality=2.16G cost=497656619
+| | tuple-ids=23 row-size=16B cardinality=2.16G cost=558351719
| | in pipelines: 20(GETNEXT)
| |
| 19:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales]
-| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| partition predicates: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=145.67GB
+| predicates: tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk IS NOT NULL
| runtime filters: RF009[min_max] -> tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk, RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk, RF010[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
| stored statistics:
| table: rows=2.16G size=145.75GB
-| partitions: 1824/1824 rows=2.16G
+| partitions: 1823/1823 rows=2.16G
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=3.37M
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=21 row-size=16B cardinality=2.16G cost=497656619
+| tuple-ids=21 row-size=16B cardinality=2.16G cost=558205026
| in pipelines: 19(GETNEXT)
|
01:UNION
@@ -274,7 +282,7 @@ PLAN-ROOT SINK
| | output: sum(SALES_PRICE), sum(PROFIT), sum(RETURN_AMT), sum(NET_LOSS)
| | group by: tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_id
| | mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB thread-reservation=0
-| | tuple-ids=19 row-size=92B cardinality=36.00K cost=2388021581
+| | tuple-ids=19 row-size=92B cardinality=36.00K cost=2362066452
| | in pipelines: 17(GETNEXT), 11(OPEN), 12(OPEN)
| |
| 16:HASH JOIN [INNER JOIN]
@@ -282,17 +290,19 @@ PLAN-ROOT SINK
| | fk/pk conjuncts: assumed fk/pk
| | runtime filters: RF004[bloom] <- tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_sk, RF005[min_max] <- tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_sk
| | mem-estimate=2.88MB mem-reservation=2.88MB spill-buffer=128.00KB thread-reservation=0
-| | tuple-ids=14,15,17 row-size=64B cardinality=4.75G cost=2080027968
+| | tuple-ids=14,15,17 row-size=64B cardinality=4.70G cost=2057419888
| | in pipelines: 11(GETNEXT), 12(GETNEXT), 15(OPEN)
| |
| |--15:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_page]
| | HDFS partitions=1/1 files=1 size=2.24MB
+| | predicates: tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_sk IS NOT NULL
| | stored statistics:
| | table: rows=36.00K size=2.24MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=36.00K
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=17 row-size=32B cardinality=36.00K cost=10368
+| | tuple-ids=17 row-size=32B cardinality=36.00K cost=11379
| | in pipelines: 15(GETNEXT)
| |
| 14:HASH JOIN [INNER JOIN]
@@ -300,56 +310,62 @@ PLAN-ROOT SINK
| | fk/pk conjuncts: assumed fk/pk
| | runtime filters: RF006[bloom] <- tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=14,15 row-size=32B cardinality=4.75G cost=2079999273
+| | tuple-ids=14,15 row-size=32B cardinality=4.70G cost=2057391193
| | in pipelines: 11(GETNEXT), 12(GETNEXT), 13(OPEN)
| |
| |--13:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=15 row-size=8B cardinality=7.30K cost=12520
+| | tuple-ids=15 row-size=8B cardinality=7.30K cost=14573
| | in pipelines: 13(GETNEXT)
| |
| 10:UNION
| | mem-estimate=0B mem-reservation=0B thread-reservation=0
-| | tuple-ids=14 row-size=24B cardinality=4.75G cost=483668226
+| | tuple-ids=14 row-size=24B cardinality=4.70G cost=478411085
| | in pipelines: 11(GETNEXT), 12(GETNEXT)
| |
| |--12:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_returns]
+| | partition predicates: tpcds_partitioned_parquet_snap.catalog_returns.cr_returned_date_sk IS NOT NULL
| | HDFS partitions=2060/2060 files=2060 size=32.77GB
+| | predicates: tpcds_partitioned_parquet_snap.catalog_returns.cr_catalog_page_sk IS NOT NULL
| | runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.catalog_returns.cr_catalog_page_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.catalog_returns.cr_catalog_page_sk, RF006[bloom] -> tpcds_partitioned_parquet_snap.catalog_returns.cr_returned_date_sk
| | stored statistics:
| | table: rows=432.02M size=32.77GB
| | partitions: 2060/2060 rows=432.02M
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=416.82K
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_returns.cr_catalog_page_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
-| | tuple-ids=12 row-size=16B cardinality=432.02M cost=99536714
+| | tuple-ids=12 row-size=16B cardinality=423.37M cost=111676392
| | in pipelines: 12(GETNEXT)
| |
| 11:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales]
-| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| partition predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk IS NOT NULL
+| HDFS partitions=1830/1831 files=1830 size=280.81GB
+| predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_catalog_page_sk IS NOT NULL
| runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_catalog_page_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_catalog_page_sk, RF006[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
| stored statistics:
| table: rows=4.32G size=280.96GB
-| partitions: 1831/1831 rows=4.32G
+| partitions: 1830/1830 rows=4.30G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=21.52M
-| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=10 row-size=16B cardinality=4.32G cost=995345966
+| extrapolated-rows=disabled max-scan-range-rows=5.09M
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_catalog_page_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=10 row-size=16B cardinality=4.28G cost=1111177945
| in pipelines: 11(GETNEXT)
|
09:AGGREGATE [FINALIZE]
| output: sum(SALES_PRICE), sum(PROFIT), sum(RETURN_AMT), sum(NET_LOSS)
| group by: tpcds_partitioned_parquet_snap.store.s_store_id
| mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=9 row-size=92B cardinality=678 cost=4775723489
+| tuple-ids=9 row-size=92B cardinality=678 cost=4363330981
| in pipelines: 09(GETNEXT), 03(OPEN), 04(OPEN)
|
08:HASH JOIN [INNER JOIN]
@@ -357,17 +373,19 @@ PLAN-ROOT SINK
| fk/pk conjuncts: assumed fk/pk
| runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.store.s_store_sk, RF001[min_max] <- tpcds_partitioned_parquet_snap.store.s_store_sk
| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=4,5,7 row-size=64B cardinality=9.50G cost=4159868808
+| tuple-ids=4,5,7 row-size=64B cardinality=8.68G cost=3800656468
| in pipelines: 03(GETNEXT), 04(GETNEXT), 07(OPEN)
|
|--07:SCAN HDFS [tpcds_partitioned_parquet_snap.store]
| HDFS partitions=1/1 files=1 size=119.76KB
+| predicates: tpcds_partitioned_parquet_snap.store.s_store_sk IS NOT NULL
| stored statistics:
| table: rows=1.35K size=119.76KB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=1.35K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store.s_store_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
-| tuple-ids=7 row-size=32B cardinality=1.35K cost=388
+| tuple-ids=7 row-size=32B cardinality=1.35K cost=426
| in pipelines: 07(GETNEXT)
|
06:HASH JOIN [INNER JOIN]
@@ -375,53 +393,59 @@ PLAN-ROOT SINK
| fk/pk conjuncts: assumed fk/pk
| runtime filters: RF002[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=4,5 row-size=32B cardinality=9.50G cost=4159874763
+| tuple-ids=4,5 row-size=32B cardinality=8.68G cost=3800662423
| in pipelines: 03(GETNEXT), 04(GETNEXT), 05(OPEN)
|
|--05:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE)
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=5 row-size=8B cardinality=7.30K cost=12520
+| tuple-ids=5 row-size=8B cardinality=7.30K cost=14573
| in pipelines: 05(GETNEXT)
|
02:UNION
| mem-estimate=0B mem-reservation=0B thread-reservation=0
-| tuple-ids=4 row-size=24B cardinality=9.50G cost=967309366
+| tuple-ids=4 row-size=24B cardinality=8.68G cost=883780392
| in pipelines: 03(GETNEXT), 04(GETNEXT)
|
|--04:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns]
-| HDFS partitions=2004/2004 files=2004 size=48.14GB
+| partition predicates: tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk IS NOT NULL
+| HDFS partitions=2003/2004 files=2003 size=48.12GB
+| predicates: tpcds_partitioned_parquet_snap.store_returns.sr_store_sk IS NOT NULL
| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk
| stored statistics:
| table: rows=863.99M size=48.14GB
-| partitions: 2004/2004 rows=863.99M
+| partitions: 2003/2003 rows=833.90M
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=30.09M
-| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=2 row-size=16B cardinality=863.99M cost=199062980
+| extrapolated-rows=disabled max-scan-range-rows=793.33K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_returns.sr_store_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+| tuple-ids=2 row-size=16B cardinality=804.72M cost=215563724
| in pipelines: 04(GETNEXT)
|
03:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales.ss_store_sk IS NOT NULL
runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=0 row-size=16B cardinality=8.64G cost=1990641068
+ extrapolated-rows=disabled max-scan-range-rows=12.22M
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_store_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=0 row-size=16B cardinality=7.88G cost=2132551022
in pipelines: 03(GETNEXT)
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=1.98GB Threads=92
-Per-Host Resource Estimates: Memory=8.63GB
+Max Per-Host Resource Reservation: Memory=1.88GB Threads=91
+Per-Host Resource Estimates: Memory=8.58GB
F23:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=4.07MB mem-reservation=4.00MB thread-reservation=1
| max-parallelism=1 segment-costs=[539] cpu-comparison-result=480 [max(1 (self) vs 480 (sum children))]
@@ -509,12 +533,12 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| F17:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
| Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
| Per-Instance Resources: mem-estimate=71.00MB mem-reservation=2.12MB thread-reservation=1
-| max-parallelism=120 segment-costs=[7193577061, 25502] cpu-comparison-result=240 [max(240 (self) vs 202 (sum children))]
+| max-parallelism=120 segment-costs=[7140661374, 25502] cpu-comparison-result=240 [max(240 (self) vs 192 (sum children))]
| 27:AGGREGATE [STREAMING]
| | output: sum(SALES_PRICE), sum(PROFIT), sum(RETURN_AMT), sum(NET_LOSS)
| | group by: tpcds_partitioned_parquet_snap.web_site.web_site_id
| | mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=32 row-size=92B cardinality=3.96K cost=1855951163
+| | tuple-ids=32 row-size=92B cardinality=3.96K cost=1821633938
| | in pipelines: 19(GETNEXT), 20(GETNEXT)
| |
| 26:HASH JOIN [INNER JOIN, BROADCAST]
@@ -522,7 +546,7 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | hash predicates: WSR_WEB_SITE_SK = tpcds_partitioned_parquet_snap.web_site.web_site_sk
| | fk/pk conjuncts: assumed fk/pk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=27,28,30 row-size=64B cardinality=3.69G cost=1616607572
+| | tuple-ids=27,28,30 row-size=64B cardinality=3.63G cost=1586715733
| | in pipelines: 19(GETNEXT), 20(GETNEXT), 25(OPEN)
| |
| |--F28:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -541,15 +565,17 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | |
| | F19:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.14MB mem-reservation=16.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[26]
+| | max-parallelism=1 segment-costs=[27]
| | 25:SCAN HDFS [tpcds_partitioned_parquet_snap.web_site, RANDOM]
| | HDFS partitions=1/1 files=1 size=17.88KB
+| | predicates: tpcds_partitioned_parquet_snap.web_site.web_site_sk IS NOT NULL
| | stored statistics:
| | table: rows=66 size=17.88KB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=66
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_site.web_site_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
-| | tuple-ids=30 row-size=32B cardinality=66 cost=19
+| | tuple-ids=30 row-size=32B cardinality=66 cost=20
| | in pipelines: 25(GETNEXT)
| |
| 24:HASH JOIN [INNER JOIN, BROADCAST]
@@ -557,7 +583,7 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | hash predicates: DATE_SK = tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk
| | fk/pk conjuncts: assumed fk/pk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=27,28 row-size=32B cardinality=3.69G cost=1616607572
+| | tuple-ids=27,28 row-size=32B cardinality=3.63G cost=1586715733
| | in pipelines: 19(GETNEXT), 20(GETNEXT), 23(OPEN)
| |
| |--F29:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -576,94 +602,100 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | |
| | F18:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=1 segment-costs=[12835]
+| | max-parallelism=1 segment-costs=[14888]
| | 23:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=28 row-size=8B cardinality=7.30K cost=12520
+| | tuple-ids=28 row-size=8B cardinality=7.30K cost=14573
| | in pipelines: 23(GETNEXT)
| |
| 18:UNION
| | mem-estimate=0B mem-reservation=0B thread-reservation=0
-| | tuple-ids=27 row-size=24B cardinality=3.69G cost=375915738
+| | tuple-ids=27 row-size=24B cardinality=3.63G cost=368964878
| | in pipelines: 19(GETNEXT), 20(GETNEXT)
| |
-| |--22:HASH JOIN [RIGHT OUTER JOIN, PARTITIONED]
+| |--22:HASH JOIN [INNER JOIN, PARTITIONED]
| | | hash-table-id=06
| | | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk = tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number = tpcds_partitioned_parquet_snap.web_returns.wr_order_number
| | | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk = tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number = tpcds_partitioned_parquet_snap.web_returns.wr_order_number
| | | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| | | tuple-ids=23N,25 row-size=40B cardinality=1.53G cost=831892314
+| | | tuple-ids=23,25 row-size=40B cardinality=1.47G cost=819579752
| | | in pipelines: 20(GETNEXT), 21(OPEN)
| | |
| | |--F30:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
-| | | | Per-Instance Resources: mem-estimate=121.04MB mem-reservation=51.00MB thread-reservation=1 runtime-filters-memory=17.00MB
-| | | | max-parallelism=120 segment-costs=[263847280]
+| | | | Per-Instance Resources: mem-estimate=118.97MB mem-reservation=51.00MB thread-reservation=1 runtime-filters-memory=17.00MB
+| | | | max-parallelism=120 segment-costs=[252350347]
| | | JOIN BUILD
| | | | join-table-id=06 plan-id=07 cohort-id=03
| | | | build expressions: tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, tpcds_partitioned_parquet_snap.web_returns.wr_order_number
| | | | runtime filters: RF012[bloom] <- tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, RF013[bloom] <- tpcds_partitioned_parquet_snap.web_returns.wr_order_number, RF014[min_max] <- tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, RF015[min_max] <- tpcds_partitioned_parquet_snap.web_returns.wr_order_number
-| | | | mem-estimate=92.40MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=216002686
+| | | | mem-estimate=90.60MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=206590542
| | | |
| | | 40:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.web_returns.wr_item_sk,tpcds_partitioned_parquet_snap.web_returns.wr_order_number)]
-| | | | mem-estimate=11.64MB mem-reservation=0B thread-reservation=0
-| | | | tuple-ids=25 row-size=24B cardinality=216.00M cost=47844594
+| | | | mem-estimate=11.37MB mem-reservation=0B thread-reservation=0
+| | | | tuple-ids=25 row-size=24B cardinality=206.59M cost=45759805
| | | | in pipelines: 21(GETNEXT)
| | | |
-| | | F15:PLAN FRAGMENT [RANDOM] hosts=10 instances=60 (adjusted from 120)
+| | | F15:PLAN FRAGMENT [RANDOM] hosts=10 instances=50 (adjusted from 120)
| | | Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-| | | Per-Instance Resources: mem-estimate=29.12MB mem-reservation=256.00KB thread-reservation=1
-| | | max-parallelism=60 segment-costs=[519788863]
+| | | Per-Instance Resources: mem-estimate=29.12MB mem-reservation=32.00KB thread-reservation=1
+| | | max-parallelism=50 segment-costs=[497139479]
| | | 21:SCAN HDFS [tpcds_partitioned_parquet_snap.web_returns, RANDOM]
-| | | HDFS partitions=2114/2114 files=2114 size=16.74GB
+| | | partition predicates: tpcds_partitioned_parquet_snap.web_returns.wr_returned_date_sk IS NOT NULL
+| | | HDFS partitions=2113/2114 files=2113 size=16.73GB
| | | runtime filters: RF010[bloom] -> tpcds_partitioned_parquet_snap.web_returns.wr_returned_date_sk
| | | stored statistics:
| | | table: rows=216.00M size=16.74GB
-| | | partitions: 2114/2114 rows=216.00M
+| | | partitions: 2113/2113 rows=206.59M
| | | columns: all
-| | | extrapolated-rows=disabled max-scan-range-rows=9.41M
-| | | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
-| | | tuple-ids=25 row-size=24B cardinality=216.00M cost=74650528
+| | | extrapolated-rows=disabled max-scan-range-rows=198.66K
+| | | mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+| | | tuple-ids=25 row-size=24B cardinality=206.59M cost=71397691
| | | in pipelines: 21(GETNEXT)
| | |
| | 39:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk,tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number)]
| | | mem-estimate=12.34MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=23 row-size=16B cardinality=2.16G cost=398946083
+| | | tuple-ids=23 row-size=16B cardinality=2.16G cost=398846314
| | | in pipelines: 20(GETNEXT)
| | |
| | F14:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
| | Per-Host Shared Resources: mem-estimate=18.00MB mem-reservation=18.00MB thread-reservation=0 runtime-filters-memory=18.00MB
| | Per-Instance Resources: mem-estimate=25.38MB mem-reservation=128.00KB thread-reservation=1
-| | max-parallelism=390 segment-costs=[3836103108]
+| | max-parallelism=390 segment-costs=[3895963328]
| | 20:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
| | HDFS partitions=1824/1824 files=1824 size=145.75GB
+| | predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_web_site_sk IS NOT NULL
| | runtime filters: RF014[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk, RF015[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number, RF009[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_web_site_sk, RF012[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk, RF013[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number, RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_web_site_sk
| | stored statistics:
| | table: rows=2.16G size=145.75GB
| | partitions: 1824/1824 rows=2.16G
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.37M
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_web_site_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| | tuple-ids=23 row-size=16B cardinality=2.16G cost=497656619
+| | tuple-ids=23 row-size=16B cardinality=2.16G cost=558351719
| | in pipelines: 20(GETNEXT)
| |
| 19:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
-| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| partition predicates: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=145.67GB
+| predicates: tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk IS NOT NULL
| runtime filters: RF009[min_max] -> tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk, RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk, RF010[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
| stored statistics:
| table: rows=2.16G size=145.75GB
-| partitions: 1824/1824 rows=2.16G
+| partitions: 1823/1823 rows=2.16G
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=3.37M
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=21 row-size=16B cardinality=2.16G cost=497656619
+| tuple-ids=21 row-size=16B cardinality=2.16G cost=558205026
| in pipelines: 19(GETNEXT)
|
01:UNION
@@ -685,13 +717,13 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| |
| F08:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
| Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
-| Per-Instance Resources: mem-estimate=71.00MB mem-reservation=6.00MB thread-reservation=1
-| max-parallelism=120 segment-costs=[8137725668, 27820800] cpu-comparison-result=120 [max(120 (self) vs 22 (sum children))]
+| Per-Instance Resources: mem-estimate=71.00MB mem-reservation=5.25MB thread-reservation=1
+| max-parallelism=120 segment-costs=[8189268894, 27820800] cpu-comparison-result=120 [max(120 (self) vs 22 (sum children))]
| 17:AGGREGATE [STREAMING]
| | output: sum(SALES_PRICE), sum(PROFIT), sum(RETURN_AMT), sum(NET_LOSS)
| | group by: tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_id
| | mem-estimate=10.00MB mem-reservation=5.00MB spill-buffer=256.00KB thread-reservation=0
-| | tuple-ids=19 row-size=92B cardinality=4.32M cost=2399190826
+| | tuple-ids=19 row-size=92B cardinality=4.32M cost=2373235696
| | in pipelines: 11(GETNEXT), 12(GETNEXT)
| |
| 16:HASH JOIN [INNER JOIN, BROADCAST]
@@ -699,7 +731,7 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | hash predicates: PAGE_SK = tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_sk
| | fk/pk conjuncts: assumed fk/pk
| | mem-estimate=0B mem-reservation=0B spill-buffer=128.00KB thread-reservation=0
-| | tuple-ids=14,15,17 row-size=64B cardinality=4.75G cost=2079991968
+| | tuple-ids=14,15,17 row-size=64B cardinality=4.70G cost=2057383888
| | in pipelines: 11(GETNEXT), 12(GETNEXT), 15(OPEN)
| |
| |--F26:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -718,15 +750,17 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | |
| | F10:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.14MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=1 segment-costs=[14256]
+| | max-parallelism=1 segment-costs=[15267]
| | 15:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_page, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.24MB
+| | predicates: tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_sk IS NOT NULL
| | stored statistics:
| | table: rows=36.00K size=2.24MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=36.00K
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=17 row-size=32B cardinality=36.00K cost=10368
+| | tuple-ids=17 row-size=32B cardinality=36.00K cost=11379
| | in pipelines: 15(GETNEXT)
| |
| 14:HASH JOIN [INNER JOIN, BROADCAST]
@@ -734,7 +768,7 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | hash predicates: DATE_SK = tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | fk/pk conjuncts: assumed fk/pk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=14,15 row-size=32B cardinality=4.75G cost=2079991968
+| | tuple-ids=14,15 row-size=32B cardinality=4.70G cost=2057383888
| | in pipelines: 11(GETNEXT), 12(GETNEXT), 13(OPEN)
| |
| |--F27:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -753,47 +787,53 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | |
| | F09:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=1 segment-costs=[12835]
+| | max-parallelism=1 segment-costs=[14888]
| | 13:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=15 row-size=8B cardinality=7.30K cost=12520
+| | tuple-ids=15 row-size=8B cardinality=7.30K cost=14573
| | in pipelines: 13(GETNEXT)
| |
| 10:UNION
| | mem-estimate=0B mem-reservation=0B thread-reservation=0
-| | tuple-ids=14 row-size=24B cardinality=4.75G cost=483668226
+| | tuple-ids=14 row-size=24B cardinality=4.70G cost=478411085
| | in pipelines: 11(GETNEXT), 12(GETNEXT)
| |
| |--12:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_returns, RANDOM]
+| | partition predicates: tpcds_partitioned_parquet_snap.catalog_returns.cr_returned_date_sk IS NOT NULL
| | HDFS partitions=2060/2060 files=2060 size=32.77GB
+| | predicates: tpcds_partitioned_parquet_snap.catalog_returns.cr_catalog_page_sk IS NOT NULL
| | runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.catalog_returns.cr_catalog_page_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.catalog_returns.cr_catalog_page_sk, RF006[bloom] -> tpcds_partitioned_parquet_snap.catalog_returns.cr_returned_date_sk
| | stored statistics:
| | table: rows=432.02M size=32.77GB
| | partitions: 2060/2060 rows=432.02M
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=416.82K
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_returns.cr_catalog_page_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
-| | tuple-ids=12 row-size=16B cardinality=432.02M cost=99536714
+| | tuple-ids=12 row-size=16B cardinality=423.37M cost=111676392
| | in pipelines: 12(GETNEXT)
| |
| 11:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
-| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| partition predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk IS NOT NULL
+| HDFS partitions=1830/1831 files=1830 size=280.81GB
+| predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_catalog_page_sk IS NOT NULL
| runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_catalog_page_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_catalog_page_sk, RF006[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
| stored statistics:
| table: rows=4.32G size=280.96GB
-| partitions: 1831/1831 rows=4.32G
+| partitions: 1830/1830 rows=4.30G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=21.52M
-| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=10 row-size=16B cardinality=4.32G cost=995345966
+| extrapolated-rows=disabled max-scan-range-rows=5.09M
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_catalog_page_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=10 row-size=16B cardinality=4.28G cost=1111177945
| in pipelines: 11(GETNEXT)
|
34:AGGREGATE [FINALIZE]
@@ -810,13 +850,13 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
|
F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
-Per-Instance Resources: mem-estimate=71.00MB mem-reservation=10.00MB thread-reservation=1
-max-parallelism=120 segment-costs=[16252682173, 523958] cpu-comparison-result=120 [max(120 (self) vs 22 (sum children))]
+Per-Instance Resources: mem-estimate=71.00MB mem-reservation=2.50MB thread-reservation=1
+max-parallelism=120 segment-costs=[15196746709, 523958] cpu-comparison-result=120 [max(120 (self) vs 22 (sum children))]
09:AGGREGATE [STREAMING]
| output: sum(SALES_PRICE), sum(PROFIT), sum(RETURN_AMT), sum(NET_LOSS)
| group by: tpcds_partitioned_parquet_snap.store.s_store_id
| mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=9 row-size=92B cardinality=81.36K cost=4775933843
+| tuple-ids=9 row-size=92B cardinality=81.36K cost=4363541335
| in pipelines: 03(GETNEXT), 04(GETNEXT)
|
08:HASH JOIN [INNER JOIN, BROADCAST]
@@ -824,7 +864,7 @@ max-parallelism=120 segment-costs=[16252682173, 523958] cpu-comparison-result=12
| hash predicates: STORE_SK = tpcds_partitioned_parquet_snap.store.s_store_sk
| fk/pk conjuncts: assumed fk/pk
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=4,5,7 row-size=64B cardinality=9.50G cost=4159867458
+| tuple-ids=4,5,7 row-size=64B cardinality=8.68G cost=3800655118
| in pipelines: 03(GETNEXT), 04(GETNEXT), 07(OPEN)
|
|--F24:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -843,15 +883,17 @@ max-parallelism=120 segment-costs=[16252682173, 523958] cpu-comparison-result=12
| |
| F04:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=16.14MB mem-reservation=16.00KB thread-reservation=1
-| max-parallelism=1 segment-costs=[533]
+| max-parallelism=1 segment-costs=[571]
| 07:SCAN HDFS [tpcds_partitioned_parquet_snap.store, RANDOM]
| HDFS partitions=1/1 files=1 size=119.76KB
+| predicates: tpcds_partitioned_parquet_snap.store.s_store_sk IS NOT NULL
| stored statistics:
| table: rows=1.35K size=119.76KB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=1.35K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store.s_store_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
-| tuple-ids=7 row-size=32B cardinality=1.35K cost=388
+| tuple-ids=7 row-size=32B cardinality=1.35K cost=426
| in pipelines: 07(GETNEXT)
|
06:HASH JOIN [INNER JOIN, BROADCAST]
@@ -859,7 +901,7 @@ max-parallelism=120 segment-costs=[16252682173, 523958] cpu-comparison-result=12
| hash predicates: DATE_SK = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| fk/pk conjuncts: assumed fk/pk
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=4,5 row-size=32B cardinality=9.50G cost=4159867458
+| tuple-ids=4,5 row-size=32B cardinality=8.68G cost=3800655118
| in pipelines: 03(GETNEXT), 04(GETNEXT), 05(OPEN)
|
|--F25:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -878,51 +920,57 @@ max-parallelism=120 segment-costs=[16252682173, 523958] cpu-comparison-result=12
| |
| F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=16.05MB mem-reservation=1.00MB thread-reservation=1
-| max-parallelism=1 segment-costs=[12835]
+| max-parallelism=1 segment-costs=[14888]
| 05:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE)
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=5 row-size=8B cardinality=7.30K cost=12520
+| tuple-ids=5 row-size=8B cardinality=7.30K cost=14573
| in pipelines: 05(GETNEXT)
|
02:UNION
| mem-estimate=0B mem-reservation=0B thread-reservation=0
-| tuple-ids=4 row-size=24B cardinality=9.50G cost=967309366
+| tuple-ids=4 row-size=24B cardinality=8.68G cost=883780392
| in pipelines: 03(GETNEXT), 04(GETNEXT)
|
|--04:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns, RANDOM]
-| HDFS partitions=2004/2004 files=2004 size=48.14GB
+| partition predicates: tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk IS NOT NULL
+| HDFS partitions=2003/2004 files=2003 size=48.12GB
+| predicates: tpcds_partitioned_parquet_snap.store_returns.sr_store_sk IS NOT NULL
| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk
| stored statistics:
| table: rows=863.99M size=48.14GB
-| partitions: 2004/2004 rows=863.99M
+| partitions: 2003/2003 rows=833.90M
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=30.09M
-| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=2 row-size=16B cardinality=863.99M cost=199062980
+| extrapolated-rows=disabled max-scan-range-rows=793.33K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_returns.sr_store_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+| tuple-ids=2 row-size=16B cardinality=804.72M cost=215563724
| in pipelines: 04(GETNEXT)
|
03:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales.ss_store_sk IS NOT NULL
runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=0 row-size=16B cardinality=8.64G cost=1990641068
+ extrapolated-rows=disabled max-scan-range-rows=12.22M
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_store_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=0 row-size=16B cardinality=7.88G cost=2132551022
in pipelines: 03(GETNEXT)
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=1.98GB Threads=92
-Per-Host Resource Estimates: Memory=8.63GB
+Max Per-Host Resource Reservation: Memory=1.88GB Threads=91
+Per-Host Resource Estimates: Memory=8.58GB
F23:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=4.07MB mem-reservation=4.00MB thread-reservation=1
| max-parallelism=1 segment-costs=[539] cpu-comparison-result=480 [max(1 (self) vs 480 (sum children))]
@@ -1010,12 +1058,12 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| F17:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
| Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
| Per-Instance Resources: mem-estimate=71.00MB mem-reservation=2.12MB thread-reservation=1
-| max-parallelism=120 segment-costs=[7193577061, 25502] cpu-comparison-result=240 [max(240 (self) vs 202 (sum children))]
+| max-parallelism=120 segment-costs=[7140661374, 25502] cpu-comparison-result=240 [max(240 (self) vs 192 (sum children))]
| 27:AGGREGATE [STREAMING]
| | output: sum(SALES_PRICE), sum(PROFIT), sum(RETURN_AMT), sum(NET_LOSS)
| | group by: tpcds_partitioned_parquet_snap.web_site.web_site_id
| | mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=32 row-size=92B cardinality=3.96K cost=1855951163
+| | tuple-ids=32 row-size=92B cardinality=3.96K cost=1821633938
| | in pipelines: 19(GETNEXT), 20(GETNEXT)
| |
| 26:HASH JOIN [INNER JOIN, BROADCAST]
@@ -1023,7 +1071,7 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | hash predicates: WSR_WEB_SITE_SK = tpcds_partitioned_parquet_snap.web_site.web_site_sk
| | fk/pk conjuncts: assumed fk/pk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=27,28,30 row-size=64B cardinality=3.69G cost=1616607572
+| | tuple-ids=27,28,30 row-size=64B cardinality=3.63G cost=1586715733
| | in pipelines: 19(GETNEXT), 20(GETNEXT), 25(OPEN)
| |
| |--F28:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -1042,15 +1090,17 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | |
| | F19:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.14MB mem-reservation=16.00KB thread-reservation=1
-| | max-parallelism=1 segment-costs=[26]
+| | max-parallelism=1 segment-costs=[27]
| | 25:SCAN HDFS [tpcds_partitioned_parquet_snap.web_site, RANDOM]
| | HDFS partitions=1/1 files=1 size=17.88KB
+| | predicates: tpcds_partitioned_parquet_snap.web_site.web_site_sk IS NOT NULL
| | stored statistics:
| | table: rows=66 size=17.88KB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=66
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_site.web_site_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
-| | tuple-ids=30 row-size=32B cardinality=66 cost=19
+| | tuple-ids=30 row-size=32B cardinality=66 cost=20
| | in pipelines: 25(GETNEXT)
| |
| 24:HASH JOIN [INNER JOIN, BROADCAST]
@@ -1058,7 +1108,7 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | hash predicates: DATE_SK = tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk
| | fk/pk conjuncts: assumed fk/pk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=27,28 row-size=32B cardinality=3.69G cost=1616607572
+| | tuple-ids=27,28 row-size=32B cardinality=3.63G cost=1586715733
| | in pipelines: 19(GETNEXT), 20(GETNEXT), 23(OPEN)
| |
| |--F29:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -1077,94 +1127,100 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | |
| | F18:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=1 segment-costs=[12835]
+| | max-parallelism=1 segment-costs=[14888]
| | 23:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_1.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_1.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim_1.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=28 row-size=8B cardinality=7.30K cost=12520
+| | tuple-ids=28 row-size=8B cardinality=7.30K cost=14573
| | in pipelines: 23(GETNEXT)
| |
| 18:UNION
| | mem-estimate=0B mem-reservation=0B thread-reservation=0
-| | tuple-ids=27 row-size=24B cardinality=3.69G cost=375915738
+| | tuple-ids=27 row-size=24B cardinality=3.63G cost=368964878
| | in pipelines: 19(GETNEXT), 20(GETNEXT)
| |
-| |--22:HASH JOIN [RIGHT OUTER JOIN, PARTITIONED]
+| |--22:HASH JOIN [INNER JOIN, PARTITIONED]
| | | hash-table-id=06
| | | hash predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk = tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number = tpcds_partitioned_parquet_snap.web_returns.wr_order_number
| | | fk/pk conjuncts: tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk = tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number = tpcds_partitioned_parquet_snap.web_returns.wr_order_number
| | | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| | | tuple-ids=23N,25 row-size=40B cardinality=1.53G cost=831892314
+| | | tuple-ids=23,25 row-size=40B cardinality=1.47G cost=819579752
| | | in pipelines: 20(GETNEXT), 21(OPEN)
| | |
| | |--F30:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
-| | | | Per-Instance Resources: mem-estimate=121.04MB mem-reservation=51.00MB thread-reservation=1 runtime-filters-memory=17.00MB
-| | | | max-parallelism=120 segment-costs=[263847280]
+| | | | Per-Instance Resources: mem-estimate=118.97MB mem-reservation=51.00MB thread-reservation=1 runtime-filters-memory=17.00MB
+| | | | max-parallelism=120 segment-costs=[252350347]
| | | JOIN BUILD
| | | | join-table-id=06 plan-id=07 cohort-id=03
| | | | build expressions: tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, tpcds_partitioned_parquet_snap.web_returns.wr_order_number
| | | | runtime filters: RF012[bloom] <- tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, RF013[bloom] <- tpcds_partitioned_parquet_snap.web_returns.wr_order_number, RF014[min_max] <- tpcds_partitioned_parquet_snap.web_returns.wr_item_sk, RF015[min_max] <- tpcds_partitioned_parquet_snap.web_returns.wr_order_number
-| | | | mem-estimate=92.40MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=216002686
+| | | | mem-estimate=90.60MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=206590542
| | | |
| | | 40:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.web_returns.wr_item_sk,tpcds_partitioned_parquet_snap.web_returns.wr_order_number)]
-| | | | mem-estimate=11.64MB mem-reservation=0B thread-reservation=0
-| | | | tuple-ids=25 row-size=24B cardinality=216.00M cost=47844594
+| | | | mem-estimate=11.37MB mem-reservation=0B thread-reservation=0
+| | | | tuple-ids=25 row-size=24B cardinality=206.59M cost=45759805
| | | | in pipelines: 21(GETNEXT)
| | | |
-| | | F15:PLAN FRAGMENT [RANDOM] hosts=10 instances=60 (adjusted from 120)
+| | | F15:PLAN FRAGMENT [RANDOM] hosts=10 instances=50 (adjusted from 120)
| | | Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
-| | | Per-Instance Resources: mem-estimate=29.12MB mem-reservation=256.00KB thread-reservation=1
-| | | max-parallelism=60 segment-costs=[519788863]
+| | | Per-Instance Resources: mem-estimate=29.12MB mem-reservation=32.00KB thread-reservation=1
+| | | max-parallelism=50 segment-costs=[497139479]
| | | 21:SCAN HDFS [tpcds_partitioned_parquet_snap.web_returns, RANDOM]
-| | | HDFS partitions=2114/2114 files=2114 size=16.74GB
+| | | partition predicates: tpcds_partitioned_parquet_snap.web_returns.wr_returned_date_sk IS NOT NULL
+| | | HDFS partitions=2113/2114 files=2113 size=16.73GB
| | | runtime filters: RF010[bloom] -> tpcds_partitioned_parquet_snap.web_returns.wr_returned_date_sk
| | | stored statistics:
| | | table: rows=216.00M size=16.74GB
-| | | partitions: 2114/2114 rows=216.00M
+| | | partitions: 2113/2113 rows=206.59M
| | | columns: all
-| | | extrapolated-rows=disabled max-scan-range-rows=9.41M
-| | | mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
-| | | tuple-ids=25 row-size=24B cardinality=216.00M cost=74650528
+| | | extrapolated-rows=disabled max-scan-range-rows=198.66K
+| | | mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+| | | tuple-ids=25 row-size=24B cardinality=206.59M cost=71397691
| | | in pipelines: 21(GETNEXT)
| | |
| | 39:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk,tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number)]
| | | mem-estimate=12.34MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=23 row-size=16B cardinality=2.16G cost=398946083
+| | | tuple-ids=23 row-size=16B cardinality=2.16G cost=398846314
| | | in pipelines: 20(GETNEXT)
| | |
| | F14:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
| | Per-Host Shared Resources: mem-estimate=18.00MB mem-reservation=18.00MB thread-reservation=0 runtime-filters-memory=18.00MB
| | Per-Instance Resources: mem-estimate=25.38MB mem-reservation=128.00KB thread-reservation=1
-| | max-parallelism=390 segment-costs=[3836103108]
+| | max-parallelism=390 segment-costs=[3895963328]
| | 20:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
| | HDFS partitions=1824/1824 files=1824 size=145.75GB
+| | predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_web_site_sk IS NOT NULL
| | runtime filters: RF014[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk, RF015[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number, RF009[min_max] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_web_site_sk, RF012[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_item_sk, RF013[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_order_number, RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales_0.ws_web_site_sk
| | stored statistics:
| | table: rows=2.16G size=145.75GB
| | partitions: 1824/1824 rows=2.16G
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=3.37M
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_sales_0.ws_web_site_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| | tuple-ids=23 row-size=16B cardinality=2.16G cost=497656619
+| | tuple-ids=23 row-size=16B cardinality=2.16G cost=558351719
| | in pipelines: 20(GETNEXT)
| |
| 19:SCAN HDFS [tpcds_partitioned_parquet_snap.web_sales, RANDOM]
-| HDFS partitions=1824/1824 files=1824 size=145.75GB
+| partition predicates: tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk IS NOT NULL
+| HDFS partitions=1823/1824 files=1823 size=145.67GB
+| predicates: tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk IS NOT NULL
| runtime filters: RF009[min_max] -> tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk, RF008[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk, RF010[bloom] -> tpcds_partitioned_parquet_snap.web_sales.ws_sold_date_sk
| stored statistics:
| table: rows=2.16G size=145.75GB
-| partitions: 1824/1824 rows=2.16G
+| partitions: 1823/1823 rows=2.16G
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=3.37M
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.web_sales.ws_web_site_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=21 row-size=16B cardinality=2.16G cost=497656619
+| tuple-ids=21 row-size=16B cardinality=2.16G cost=558205026
| in pipelines: 19(GETNEXT)
|
01:UNION
@@ -1186,13 +1242,13 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| |
| F08:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
| Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
-| Per-Instance Resources: mem-estimate=71.00MB mem-reservation=6.00MB thread-reservation=1
-| max-parallelism=120 segment-costs=[8137725668, 27820800] cpu-comparison-result=120 [max(120 (self) vs 22 (sum children))]
+| Per-Instance Resources: mem-estimate=71.00MB mem-reservation=5.25MB thread-reservation=1
+| max-parallelism=120 segment-costs=[8189268894, 27820800] cpu-comparison-result=120 [max(120 (self) vs 22 (sum children))]
| 17:AGGREGATE [STREAMING]
| | output: sum(SALES_PRICE), sum(PROFIT), sum(RETURN_AMT), sum(NET_LOSS)
| | group by: tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_id
| | mem-estimate=10.00MB mem-reservation=5.00MB spill-buffer=256.00KB thread-reservation=0
-| | tuple-ids=19 row-size=92B cardinality=4.32M cost=2399190826
+| | tuple-ids=19 row-size=92B cardinality=4.32M cost=2373235696
| | in pipelines: 11(GETNEXT), 12(GETNEXT)
| |
| 16:HASH JOIN [INNER JOIN, BROADCAST]
@@ -1200,7 +1256,7 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | hash predicates: PAGE_SK = tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_sk
| | fk/pk conjuncts: assumed fk/pk
| | mem-estimate=0B mem-reservation=0B spill-buffer=128.00KB thread-reservation=0
-| | tuple-ids=14,15,17 row-size=64B cardinality=4.75G cost=2079991968
+| | tuple-ids=14,15,17 row-size=64B cardinality=4.70G cost=2057383888
| | in pipelines: 11(GETNEXT), 12(GETNEXT), 15(OPEN)
| |
| |--F26:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -1219,15 +1275,17 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | |
| | F10:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.14MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=1 segment-costs=[14256]
+| | max-parallelism=1 segment-costs=[15267]
| | 15:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_page, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.24MB
+| | predicates: tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_sk IS NOT NULL
| | stored statistics:
| | table: rows=36.00K size=2.24MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=36.00K
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_page.cp_catalog_page_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=17 row-size=32B cardinality=36.00K cost=10368
+| | tuple-ids=17 row-size=32B cardinality=36.00K cost=11379
| | in pipelines: 15(GETNEXT)
| |
| 14:HASH JOIN [INNER JOIN, BROADCAST]
@@ -1235,7 +1293,7 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | hash predicates: DATE_SK = tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk
| | fk/pk conjuncts: assumed fk/pk
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=14,15 row-size=32B cardinality=4.75G cost=2079991968
+| | tuple-ids=14,15 row-size=32B cardinality=4.70G cost=2057383888
| | in pipelines: 11(GETNEXT), 12(GETNEXT), 13(OPEN)
| |
| |--F27:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -1254,47 +1312,53 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
| | |
| | F09:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| | Per-Instance Resources: mem-estimate=16.05MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=1 segment-costs=[12835]
+| | max-parallelism=1 segment-costs=[14888]
| | 13:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| | HDFS partitions=1/1 files=1 size=2.17MB
-| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE)
+| | predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | stored statistics:
| | table: rows=73.05K size=2.17MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=73.05K
| | parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE)
-| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim_0.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim_0.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim_0.d_date_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=15 row-size=8B cardinality=7.30K cost=12520
+| | tuple-ids=15 row-size=8B cardinality=7.30K cost=14573
| | in pipelines: 13(GETNEXT)
| |
| 10:UNION
| | mem-estimate=0B mem-reservation=0B thread-reservation=0
-| | tuple-ids=14 row-size=24B cardinality=4.75G cost=483668226
+| | tuple-ids=14 row-size=24B cardinality=4.70G cost=478411085
| | in pipelines: 11(GETNEXT), 12(GETNEXT)
| |
| |--12:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_returns, RANDOM]
+| | partition predicates: tpcds_partitioned_parquet_snap.catalog_returns.cr_returned_date_sk IS NOT NULL
| | HDFS partitions=2060/2060 files=2060 size=32.77GB
+| | predicates: tpcds_partitioned_parquet_snap.catalog_returns.cr_catalog_page_sk IS NOT NULL
| | runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.catalog_returns.cr_catalog_page_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.catalog_returns.cr_catalog_page_sk, RF006[bloom] -> tpcds_partitioned_parquet_snap.catalog_returns.cr_returned_date_sk
| | stored statistics:
| | table: rows=432.02M size=32.77GB
| | partitions: 2060/2060 rows=432.02M
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=416.82K
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_returns.cr_catalog_page_sk IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
-| | tuple-ids=12 row-size=16B cardinality=432.02M cost=99536714
+| | tuple-ids=12 row-size=16B cardinality=423.37M cost=111676392
| | in pipelines: 12(GETNEXT)
| |
| 11:SCAN HDFS [tpcds_partitioned_parquet_snap.catalog_sales, RANDOM]
-| HDFS partitions=1831/1831 files=1831 size=280.96GB
+| partition predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk IS NOT NULL
+| HDFS partitions=1830/1831 files=1830 size=280.81GB
+| predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_catalog_page_sk IS NOT NULL
| runtime filters: RF005[min_max] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_catalog_page_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_catalog_page_sk, RF006[bloom] -> tpcds_partitioned_parquet_snap.catalog_sales.cs_sold_date_sk
| stored statistics:
| table: rows=4.32G size=280.96GB
-| partitions: 1831/1831 rows=4.32G
+| partitions: 1830/1830 rows=4.30G
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=21.52M
-| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=10 row-size=16B cardinality=4.32G cost=995345966
+| extrapolated-rows=disabled max-scan-range-rows=5.09M
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.catalog_sales.cs_catalog_page_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=256.00KB thread-reservation=0
+| tuple-ids=10 row-size=16B cardinality=4.28G cost=1111177945
| in pipelines: 11(GETNEXT)
|
34:AGGREGATE [FINALIZE]
@@ -1311,13 +1375,13 @@ max-parallelism=120 segment-costs=[86121, 4572835, 4190, 509107, 1136665] cpu-co
|
F02:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
Per-Host Shared Resources: mem-estimate=2.00MB mem-reservation=2.00MB thread-reservation=0 runtime-filters-memory=2.00MB
-Per-Instance Resources: mem-estimate=71.00MB mem-reservation=10.00MB thread-reservation=1
-max-parallelism=120 segment-costs=[16252682173, 523958] cpu-comparison-result=120 [max(120 (self) vs 22 (sum children))]
+Per-Instance Resources: mem-estimate=71.00MB mem-reservation=2.50MB thread-reservation=1
+max-parallelism=120 segment-costs=[15196746709, 523958] cpu-comparison-result=120 [max(120 (self) vs 22 (sum children))]
09:AGGREGATE [STREAMING]
| output: sum(SALES_PRICE), sum(PROFIT), sum(RETURN_AMT), sum(NET_LOSS)
| group by: tpcds_partitioned_parquet_snap.store.s_store_id
| mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=9 row-size=92B cardinality=81.36K cost=4775933843
+| tuple-ids=9 row-size=92B cardinality=81.36K cost=4363541335
| in pipelines: 03(GETNEXT), 04(GETNEXT)
|
08:HASH JOIN [INNER JOIN, BROADCAST]
@@ -1325,7 +1389,7 @@ max-parallelism=120 segment-costs=[16252682173, 523958] cpu-comparison-result=12
| hash predicates: STORE_SK = tpcds_partitioned_parquet_snap.store.s_store_sk
| fk/pk conjuncts: assumed fk/pk
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=4,5,7 row-size=64B cardinality=9.50G cost=4159867458
+| tuple-ids=4,5,7 row-size=64B cardinality=8.68G cost=3800655118
| in pipelines: 03(GETNEXT), 04(GETNEXT), 07(OPEN)
|
|--F24:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -1344,15 +1408,17 @@ max-parallelism=120 segment-costs=[16252682173, 523958] cpu-comparison-result=12
| |
| F04:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=16.14MB mem-reservation=16.00KB thread-reservation=1
-| max-parallelism=1 segment-costs=[533]
+| max-parallelism=1 segment-costs=[571]
| 07:SCAN HDFS [tpcds_partitioned_parquet_snap.store, RANDOM]
| HDFS partitions=1/1 files=1 size=119.76KB
+| predicates: tpcds_partitioned_parquet_snap.store.s_store_sk IS NOT NULL
| stored statistics:
| table: rows=1.35K size=119.76KB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=1.35K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store.s_store_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=0
-| tuple-ids=7 row-size=32B cardinality=1.35K cost=388
+| tuple-ids=7 row-size=32B cardinality=1.35K cost=426
| in pipelines: 07(GETNEXT)
|
06:HASH JOIN [INNER JOIN, BROADCAST]
@@ -1360,7 +1426,7 @@ max-parallelism=120 segment-costs=[16252682173, 523958] cpu-comparison-result=12
| hash predicates: DATE_SK = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| fk/pk conjuncts: assumed fk/pk
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=4,5 row-size=32B cardinality=9.50G cost=4159867458
+| tuple-ids=4,5 row-size=32B cardinality=8.68G cost=3800655118
| in pipelines: 03(GETNEXT), 04(GETNEXT), 05(OPEN)
|
|--F25:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -1379,46 +1445,52 @@ max-parallelism=120 segment-costs=[16252682173, 523958] cpu-comparison-result=12
| |
| F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=16.05MB mem-reservation=1.00MB thread-reservation=1
-| max-parallelism=1 segment-costs=[12835]
+| max-parallelism=1 segment-costs=[14888]
| 05:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE)
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_date <= CAST('2000-08-19' AS DATE) + INTERVAL CAST(14 AS INT) DAY, tpcds_partitioned_parquet_snap.date_dim.d_date >= CAST('2000-08-19' AS DATE), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=5 row-size=8B cardinality=7.30K cost=12520
+| tuple-ids=5 row-size=8B cardinality=7.30K cost=14573
| in pipelines: 05(GETNEXT)
|
02:UNION
| mem-estimate=0B mem-reservation=0B thread-reservation=0
-| tuple-ids=4 row-size=24B cardinality=9.50G cost=967309366
+| tuple-ids=4 row-size=24B cardinality=8.68G cost=883780392
| in pipelines: 03(GETNEXT), 04(GETNEXT)
|
|--04:SCAN HDFS [tpcds_partitioned_parquet_snap.store_returns, RANDOM]
-| HDFS partitions=2004/2004 files=2004 size=48.14GB
+| partition predicates: tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk IS NOT NULL
+| HDFS partitions=2003/2004 files=2003 size=48.12GB
+| predicates: tpcds_partitioned_parquet_snap.store_returns.sr_store_sk IS NOT NULL
| runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_store_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_returns.sr_returned_date_sk
| stored statistics:
| table: rows=863.99M size=48.14GB
-| partitions: 2004/2004 rows=863.99M
+| partitions: 2003/2003 rows=833.90M
| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=30.09M
-| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=2 row-size=16B cardinality=863.99M cost=199062980
+| extrapolated-rows=disabled max-scan-range-rows=793.33K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_returns.sr_store_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
+| tuple-ids=2 row-size=16B cardinality=804.72M cost=215563724
| in pipelines: 04(GETNEXT)
|
03:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales.ss_store_sk IS NOT NULL
runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_store_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=0 row-size=16B cardinality=8.64G cost=1990641068
+ extrapolated-rows=disabled max-scan-range-rows=12.22M
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_store_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=0 row-size=16B cardinality=7.88G cost=2132551022
in pipelines: 03(GETNEXT)
====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q06.test b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q06.test
index eaeb0ac3e..31a980547 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q06.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q06.test
@@ -24,11 +24,11 @@ select a.ca_state state, count(*) cnt
order by cnt, a.ca_state
limit 100
---- PLAN
-Max Per-Host Resource Reservation: Memory=152.94MB Threads=1
+Max Per-Host Resource Reservation: Memory=145.44MB Threads=1
Per-Host Resource Estimates: Memory=2.86GB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Instance Resources: mem-estimate=2.86GB mem-reservation=152.94MB thread-reservation=1 runtime-filters-memory=36.00MB
-| max-parallelism=1 segment-costs=[17041, 0, 72019, 316624331, 6, 10]
+| Per-Instance Resources: mem-estimate=2.86GB mem-reservation=145.44MB thread-reservation=1 runtime-filters-memory=36.00MB
+| max-parallelism=1 segment-costs=[17041, 0, 166538, 310809299, 6, 10]
PLAN-ROOT SINK
| output exprs: tpcds_partitioned_parquet_snap.customer_address.ca_state, count()
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=10
@@ -44,101 +44,105 @@ PLAN-ROOT SINK
| group by: tpcds_partitioned_parquet_snap.customer_address.ca_state
| having: count() >= CAST(10 AS BIGINT)
| mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=17 row-size=22B cardinality=5 cost=38137040
+| tuple-ids=17 row-size=22B cardinality=5 cost=34775830
| in pipelines: 16(GETNEXT), 00(OPEN)
|
15:HASH JOIN [INNER JOIN]
+| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk, RF001[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| mem-estimate=1.48GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0,2,6,7,12,15,13 row-size=102B cardinality=118.89M cost=84763705
+| in pipelines: 00(GETNEXT), 13(OPEN)
+|
+|--14:HASH JOIN [INNER JOIN]
+| | hash predicates: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk = tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk = tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
+| | runtime filters: RF010[bloom] <- tpcds_partitioned_parquet_snap.customer.c_current_addr_sk, RF011[min_max] <- tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
+| | mem-estimate=1.22GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=15,13 row-size=26B cardinality=30.00M cost=39283500
+| | in pipelines: 13(GETNEXT), 12(OPEN)
+| |
+| |--12:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | predicates: tpcds_partitioned_parquet_snap.customer.c_current_addr_sk IS NOT NULL, tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer.c_current_addr_sk IS NOT NULL, tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=13 row-size=8B cardinality=30.00M cost=5142000
+| | in pipelines: 12(GETNEXT)
+| |
+| 13:SCAN HDFS [tpcds_partitioned_parquet_snap.customer_address]
+| HDFS partitions=1/1 files=1 size=307.36MB
+| predicates: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk IS NOT NULL
+| runtime filters: RF011[min_max] -> tpcds_partitioned_parquet_snap.customer_address.ca_address_sk, RF010[bloom] -> tpcds_partitioned_parquet_snap.customer_address.ca_address_sk
+| stored statistics:
+| table: rows=15.00M size=307.36MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=1.58M
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=15 row-size=18B cardinality=15.00M cost=1717500
+| in pipelines: 13(GETNEXT)
+|
+11:HASH JOIN [INNER JOIN]
| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk = tpcds_partitioned_parquet_snap.item.i_item_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk = tpcds_partitioned_parquet_snap.item.i_item_sk
-| runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.item.i_item_sk, RF001[min_max] <- tpcds_partitioned_parquet_snap.item.i_item_sk
+| runtime filters: RF002[bloom] <- tpcds_partitioned_parquet_snap.item.i_item_sk, RF003[min_max] <- tpcds_partitioned_parquet_snap.item.i_item_sk
| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=0,2,6,9,7,11,16 row-size=102B cardinality=130.38M cost=57445181
-| in pipelines: 00(GETNEXT), 11(OPEN)
+| tuple-ids=0,2,6,7,12 row-size=76B cardinality=129.52M cost=57065341
+| in pipelines: 00(GETNEXT), 07(OPEN)
|
-|--14:HASH JOIN [INNER JOIN]
+|--10:HASH JOIN [INNER JOIN]
| | hash predicates: tpcds_partitioned_parquet_snap.item.i_category = tpcds_partitioned_parquet_snap.item_0.i_category
| | fk/pk conjuncts: none
| | other join predicates: tpcds_partitioned_parquet_snap.item.i_current_price > multiply(CAST(1.2 AS DECIMAL(2,1)), avg(tpcds_partitioned_parquet_snap.item_0.i_current_price))
-| | runtime filters: RF010[bloom] <- tpcds_partitioned_parquet_snap.item_0.i_category, RF011[min_max] <- tpcds_partitioned_parquet_snap.item_0.i_category
+| | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.item_0.i_category, RF009[min_max] <- tpcds_partitioned_parquet_snap.item_0.i_category
| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=11,16 row-size=52B cardinality=360.00K cost=74476
-| | in pipelines: 11(GETNEXT), 13(OPEN)
+| | tuple-ids=7,12 row-size=52B cardinality=360.00K cost=157240
+| | in pipelines: 07(GETNEXT), 09(OPEN)
| |
-| |--13:AGGREGATE [FINALIZE]
+| |--09:AGGREGATE [FINALIZE]
| | | output: avg(tpcds_partitioned_parquet_snap.item_0.i_current_price)
| | | group by: tpcds_partitioned_parquet_snap.item_0.i_category
| | | mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | | tuple-ids=16 row-size=26B cardinality=10 cost=10556
-| | | in pipelines: 13(GETNEXT), 12(OPEN)
+| | | tuple-ids=12 row-size=26B cardinality=10 cost=105075
+| | | in pipelines: 09(GETNEXT), 08(OPEN)
| | |
-| | 12:SCAN HDFS [tpcds_partitioned_parquet_snap.item]
+| | 08:SCAN HDFS [tpcds_partitioned_parquet_snap.item]
| | HDFS partitions=1/1 files=1 size=33.54MB
-| | predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.item_0.i_category)
+| | predicates: tpcds_partitioned_parquet_snap.item_0.i_category IS NOT NULL
| | stored statistics:
| | table: rows=360.00K size=33.54MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=101.54K
-| | parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.item_0.i_category)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.item_0.i_category IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=13 row-size=22B cardinality=36.00K cost=61463
-| | in pipelines: 12(GETNEXT)
+| | tuple-ids=9 row-size=22B cardinality=359.14K cost=61463
+| | in pipelines: 08(GETNEXT)
| |
-| 11:SCAN HDFS [tpcds_partitioned_parquet_snap.item]
+| 07:SCAN HDFS [tpcds_partitioned_parquet_snap.item]
| HDFS partitions=1/1 files=1 size=33.54MB
-| predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.item.i_category)
-| runtime filters: RF011[min_max] -> tpcds_partitioned_parquet_snap.item.i_category, RF010[bloom] -> tpcds_partitioned_parquet_snap.item.i_category
+| predicates: tpcds_partitioned_parquet_snap.item.i_current_price IS NOT NULL, tpcds_partitioned_parquet_snap.item.i_category IS NOT NULL, tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
+| runtime filters: RF009[min_max] -> tpcds_partitioned_parquet_snap.item.i_category, RF008[bloom] -> tpcds_partitioned_parquet_snap.item.i_category
| stored statistics:
| table: rows=360.00K size=33.54MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=101.54K
-| parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.item.i_category)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.item.i_current_price IS NOT NULL, tpcds_partitioned_parquet_snap.item.i_category IS NOT NULL, tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=11 row-size=26B cardinality=36.00K cost=82199
-| in pipelines: 11(GETNEXT)
-|
-10:HASH JOIN [INNER JOIN]
-| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| runtime filters: RF002[bloom] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk, RF003[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| mem-estimate=1.48GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=0,2,6,9,7 row-size=50B cardinality=130.45M cost=90086864
-| in pipelines: 00(GETNEXT), 08(OPEN)
-|
-|--09:HASH JOIN [INNER JOIN]
-| | hash predicates: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk = tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk = tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
-| | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.customer.c_current_addr_sk, RF009[min_max] <- tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
-| | mem-estimate=1.22GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=9,7 row-size=26B cardinality=30.00M cost=39283500
-| | in pipelines: 08(GETNEXT), 07(OPEN)
-| |
-| |--07:SCAN HDFS [tpcds_partitioned_parquet_snap.customer]
-| | HDFS partitions=1/1 files=1 size=1.55GB
-| | stored statistics:
-| | table: rows=30.00M size=1.55GB
-| | columns: all
-| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=7 row-size=8B cardinality=30.00M cost=3456000
-| | in pipelines: 07(GETNEXT)
-| |
-| 08:SCAN HDFS [tpcds_partitioned_parquet_snap.customer_address]
-| HDFS partitions=1/1 files=1 size=307.36MB
-| runtime filters: RF009[min_max] -> tpcds_partitioned_parquet_snap.customer_address.ca_address_sk, RF008[bloom] -> tpcds_partitioned_parquet_snap.customer_address.ca_address_sk
-| stored statistics:
-| table: rows=15.00M size=307.36MB
-| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=1.58M
-| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=9 row-size=18B cardinality=15.00M cost=1296000
-| in pipelines: 08(GETNEXT)
+| tuple-ids=7 row-size=26B cardinality=358.67K cost=102431
+| in pipelines: 07(GETNEXT)
|
06:HASH JOIN [INNER JOIN]
| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| runtime filters: RF004[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=0,2,6 row-size=24B cardinality=142.10M cost=62199037
+| tuple-ids=0,2,6 row-size=24B cardinality=129.58M cost=56717105
| in pipelines: 00(GETNEXT), 01(OPEN)
|
|--05:HASH JOIN [INNER JOIN]
@@ -177,29 +181,34 @@ PLAN-ROOT SINK
| |
| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| HDFS partitions=1/1 files=1 size=2.17MB
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim.d_month_seq IS NOT NULL
| runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_month_seq, RF006[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_month_seq
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim.d_month_seq IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=2 row-size=8B cardinality=30(filtered from 73.05K) cost=8415
+| tuple-ids=2 row-size=8B cardinality=30(filtered from 73.05K) cost=12520
| in pipelines: 01(GETNEXT)
|
00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
- runtime filters: RF003[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
+ runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF003[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=31(filtered from 1824)
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=0 row-size=12B cardinality=142.10M(filtered from 8.64G) cost=24555605
+ extrapolated-rows=disabled max-scan-range-rows=12.22M est-scan-range=30(filtered from 1823)
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=0 row-size=12B cardinality=129.58M(filtered from 7.88G) cost=31072113
in pipelines: 00(GETNEXT)
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=401.81MB Threads=33
-Per-Host Resource Estimates: Memory=942MB
+Max Per-Host Resource Reservation: Memory=651.81MB Threads=33
+Per-Host Resource Estimates: Memory=1.32GB
F13:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=4.02MB mem-reservation=4.00MB thread-reservation=1
| max-parallelism=1 segment-costs=[11] cpu-comparison-result=140 [max(1 (self) vs 140 (sum children))]
@@ -236,210 +245,214 @@ max-parallelism=10 segment-costs=[3222, 6, 4] cpu-comparison-result=140 [max(10
| tuple-ids=17 row-size=22B cardinality=6.12K cost=1299
| in pipelines: 00(GETNEXT)
|
-F08:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=20 (adjusted from 120)
-Per-Instance Resources: mem-estimate=25.23MB mem-reservation=2.00MB thread-reservation=1
-max-parallelism=20 segment-costs=[197259856, 11823] cpu-comparison-result=140 [max(140 (self) vs 65 (sum children))]
+F11:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=20 (adjusted from 120)
+Per-Instance Resources: mem-estimate=32.24MB mem-reservation=2.00MB thread-reservation=1
+max-parallelism=20 segment-costs=[168174460, 11823] cpu-comparison-result=140 [max(140 (self) vs 65 (sum children))]
16:AGGREGATE [STREAMING]
| output: count()
| group by: tpcds_partitioned_parquet_snap.customer_address.ca_state
| mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=17 row-size=22B cardinality=6.12K cost=38152863
+| tuple-ids=17 row-size=22B cardinality=6.12K cost=34791653
| in pipelines: 00(GETNEXT)
|
-15:HASH JOIN [INNER JOIN, BROADCAST]
+15:HASH JOIN [INNER JOIN, PARTITIONED]
| hash-table-id=00
+| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0,2,6,7,12,15,13 row-size=102B cardinality=118.89M cost=54763705
+| in pipelines: 00(GETNEXT), 13(OPEN)
+|
+|--F14:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=20 (adjusted from 120)
+| | Per-Instance Resources: mem-estimate=101.93MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | max-parallelism=20 segment-costs=[38025000] cpu-comparison-result=40 [max(40 (self) vs 20 (sum children))]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk, RF001[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | mem-estimate=75.59MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| |
+| 30:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_sk)]
+| | mem-estimate=10.33MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=15,13 row-size=26B cardinality=30.00M cost=8025000
+| | in pipelines: 13(GETNEXT)
+| |
+| F10:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_address.ca_address_sk)] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=12.87MB mem-reservation=0B thread-reservation=1
+| max-parallelism=10 segment-costs=[93336000]
+| 14:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=01
+| | hash predicates: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk = tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk = tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=15,13 row-size=26B cardinality=30.00M cost=9283500
+| | in pipelines: 13(GETNEXT), 12(OPEN)
+| |
+| |--F15:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_address.ca_address_sk)] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=151.22MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | | max-parallelism=10 segment-costs=[34437000]
+| | JOIN BUILD
+| | | join-table-id=01 plan-id=02 cohort-id=02
+| | | build expressions: tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
+| | | runtime filters: RF010[bloom] <- tpcds_partitioned_parquet_snap.customer.c_current_addr_sk, RF011[min_max] <- tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
+| | | mem-estimate=125.11MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | |
+| | 28:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_current_addr_sk)]
+| | | mem-estimate=10.12MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=13 row-size=8B cardinality=30.00M cost=4437000
+| | | in pipelines: 12(GETNEXT)
+| | |
+| | F09:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=16.47MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[36054000]
+| | 12:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | predicates: tpcds_partitioned_parquet_snap.customer.c_current_addr_sk IS NOT NULL, tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer.c_current_addr_sk IS NOT NULL, tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=13 row-size=8B cardinality=30.00M cost=5142000
+| | in pipelines: 12(GETNEXT)
+| |
+| 27:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_address.ca_address_sk)]
+| | mem-estimate=10.21MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=15 row-size=18B cardinality=15.00M cost=2908500
+| | in pipelines: 13(GETNEXT)
+| |
+| F08:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| Per-Host Shared Resources: mem-estimate=16.00MB mem-reservation=16.00MB thread-reservation=0 runtime-filters-memory=16.00MB
+| Per-Instance Resources: mem-estimate=16.86MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=10 segment-costs=[26833500]
+| 13:SCAN HDFS [tpcds_partitioned_parquet_snap.customer_address, RANDOM]
+| HDFS partitions=1/1 files=1 size=307.36MB
+| predicates: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk IS NOT NULL
+| runtime filters: RF011[min_max] -> tpcds_partitioned_parquet_snap.customer_address.ca_address_sk, RF010[bloom] -> tpcds_partitioned_parquet_snap.customer_address.ca_address_sk
+| stored statistics:
+| table: rows=15.00M size=307.36MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=1.58M
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=15 row-size=18B cardinality=15.00M cost=1717500
+| in pipelines: 13(GETNEXT)
+|
+29:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)]
+| mem-estimate=21.23MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0,2,6,7,12 row-size=76B cardinality=129.52M cost=78619102
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Host Shared Resources: mem-estimate=18.00MB mem-reservation=18.00MB thread-reservation=0 runtime-filters-memory=18.00MB
+Per-Instance Resources: mem-estimate=23.49MB mem-reservation=512.00KB thread-reservation=1
+max-parallelism=490 segment-costs=[4877793238]
+11:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=02
| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk = tpcds_partitioned_parquet_snap.item.i_item_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk = tpcds_partitioned_parquet_snap.item.i_item_sk
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=0,2,6,9,7,11,16 row-size=102B cardinality=130.38M cost=57085181
-| in pipelines: 00(GETNEXT), 11(OPEN)
+| tuple-ids=0,2,6,7,12 row-size=76B cardinality=129.52M cost=56705341
+| in pipelines: 00(GETNEXT), 07(OPEN)
|
-|--F14:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=10
-| | Per-Instance Resources: mem-estimate=79.23MB mem-reservation=69.00MB thread-reservation=1 runtime-filters-memory=1.00MB
+|--F16:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=419.23MB mem-reservation=409.00MB thread-reservation=1 runtime-filters-memory=1.00MB
| | max-parallelism=10 segment-costs=[838440] cpu-comparison-result=14 [max(14 (self) vs 4 (sum children))]
| JOIN BUILD
-| | join-table-id=00 plan-id=01 cohort-id=01
+| | join-table-id=02 plan-id=03 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.item.i_item_sk
-| | runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.item.i_item_sk, RF001[min_max] <- tpcds_partitioned_parquet_snap.item.i_item_sk
-| | mem-estimate=68.00MB mem-reservation=68.00MB spill-buffer=2.00MB thread-reservation=0 cost=360000
+| | runtime filters: RF002[bloom] <- tpcds_partitioned_parquet_snap.item.i_item_sk, RF003[min_max] <- tpcds_partitioned_parquet_snap.item.i_item_sk
+| | mem-estimate=408.00MB mem-reservation=408.00MB spill-buffer=2.00MB thread-reservation=0 cost=360000
| |
-| 30:EXCHANGE [BROADCAST]
+| 26:EXCHANGE [BROADCAST]
| | mem-estimate=10.23MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=11,16 row-size=52B cardinality=360.00K cost=478440
-| | in pipelines: 11(GETNEXT)
+| | tuple-ids=7,12 row-size=52B cardinality=360.00K cost=478440
+| | in pipelines: 07(GETNEXT)
| |
-| F09:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+| F05:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
| Per-Instance Resources: mem-estimate=16.23MB mem-reservation=512.00KB thread-reservation=1
-| max-parallelism=4 segment-costs=[222576]
-| 14:HASH JOIN [INNER JOIN, BROADCAST]
-| | hash-table-id=01
+| max-parallelism=4 segment-costs=[325572]
+| 10:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=03
| | hash predicates: tpcds_partitioned_parquet_snap.item.i_category = tpcds_partitioned_parquet_snap.item_0.i_category
| | fk/pk conjuncts: none
| | other join predicates: tpcds_partitioned_parquet_snap.item.i_current_price > multiply(CAST(1.2 AS DECIMAL(2,1)), avg(tpcds_partitioned_parquet_snap.item_0.i_current_price))
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=11,16 row-size=52B cardinality=360.00K cost=74466
-| | in pipelines: 11(GETNEXT), 28(OPEN)
+| | tuple-ids=7,12 row-size=52B cardinality=360.00K cost=157230
+| | in pipelines: 07(GETNEXT), 24(OPEN)
| |
-| |--F15:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+| |--F17:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
| | | Per-Instance Resources: mem-estimate=2.95MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=4 segment-costs=[14] cpu-comparison-result=4 [max(4 (self) vs 4 (sum children))]
| | JOIN BUILD
-| | | join-table-id=01 plan-id=02 cohort-id=02
+| | | join-table-id=03 plan-id=04 cohort-id=03
| | | build expressions: tpcds_partitioned_parquet_snap.item_0.i_category
-| | | runtime filters: RF010[bloom] <- tpcds_partitioned_parquet_snap.item_0.i_category, RF011[min_max] <- tpcds_partitioned_parquet_snap.item_0.i_category
+| | | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.item_0.i_category, RF009[min_max] <- tpcds_partitioned_parquet_snap.item_0.i_category
| | | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0 cost=10
| | |
-| | 29:EXCHANGE [BROADCAST]
+| | 25:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=16 row-size=26B cardinality=10 cost=4
-| | | in pipelines: 28(GETNEXT)
+| | | tuple-ids=12 row-size=26B cardinality=10 cost=4
+| | | in pipelines: 24(GETNEXT)
| | |
-| | F11:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.item_0.i_category)] hosts=4 instances=4
+| | F07:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.item_0.i_category)] hosts=4 instances=4
| | Per-Instance Resources: mem-estimate=10.12MB mem-reservation=1.94MB thread-reservation=1
| | max-parallelism=4 segment-costs=[46, 0] cpu-comparison-result=4 [max(4 (self) vs 4 (sum children))]
-| | 28:AGGREGATE [FINALIZE]
+| | 24:AGGREGATE [FINALIZE]
| | | output: avg:merge(tpcds_partitioned_parquet_snap.item_0.i_current_price)
| | | group by: tpcds_partitioned_parquet_snap.item_0.i_category
| | | mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | | tuple-ids=16 row-size=26B cardinality=10 cost=37
-| | | in pipelines: 28(GETNEXT), 12(OPEN)
+| | | tuple-ids=12 row-size=26B cardinality=10 cost=37
+| | | in pipelines: 24(GETNEXT), 08(OPEN)
| | |
-| | 27:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.item_0.i_category)]
+| | 23:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.item_0.i_category)]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=15 row-size=26B cardinality=40 cost=9
-| | | in pipelines: 12(GETNEXT)
+| | | tuple-ids=11 row-size=26B cardinality=40 cost=9
+| | | in pipelines: 08(GETNEXT)
| | |
-| | F10:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+| | F06:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
| | Per-Instance Resources: mem-estimate=26.47MB mem-reservation=2.50MB thread-reservation=1
-| | max-parallelism=4 segment-costs=[72097, 87]
-| | 13:AGGREGATE [STREAMING]
+| | max-parallelism=4 segment-costs=[166617, 87]
+| | 09:AGGREGATE [STREAMING]
| | | output: avg(tpcds_partitioned_parquet_snap.item_0.i_current_price)
| | | group by: tpcds_partitioned_parquet_snap.item_0.i_category
| | | mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
-| | | tuple-ids=15 row-size=26B cardinality=40 cost=10634
-| | | in pipelines: 12(GETNEXT)
+| | | tuple-ids=11 row-size=26B cardinality=40 cost=105154
+| | | in pipelines: 08(GETNEXT)
| | |
-| | 12:SCAN HDFS [tpcds_partitioned_parquet_snap.item, RANDOM]
+| | 08:SCAN HDFS [tpcds_partitioned_parquet_snap.item, RANDOM]
| | HDFS partitions=1/1 files=1 size=33.54MB
-| | predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.item_0.i_category)
+| | predicates: tpcds_partitioned_parquet_snap.item_0.i_category IS NOT NULL
| | stored statistics:
| | table: rows=360.00K size=33.54MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=101.54K
-| | parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.item_0.i_category)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.item_0.i_category IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=13 row-size=22B cardinality=36.00K cost=61463
-| | in pipelines: 12(GETNEXT)
+| | tuple-ids=9 row-size=22B cardinality=359.14K cost=61463
+| | in pipelines: 08(GETNEXT)
| |
-| 11:SCAN HDFS [tpcds_partitioned_parquet_snap.item, RANDOM]
+| 07:SCAN HDFS [tpcds_partitioned_parquet_snap.item, RANDOM]
| HDFS partitions=1/1 files=1 size=33.54MB
-| predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.item.i_category)
-| runtime filters: RF011[min_max] -> tpcds_partitioned_parquet_snap.item.i_category, RF010[bloom] -> tpcds_partitioned_parquet_snap.item.i_category
+| predicates: tpcds_partitioned_parquet_snap.item.i_current_price IS NOT NULL, tpcds_partitioned_parquet_snap.item.i_category IS NOT NULL, tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
+| runtime filters: RF009[min_max] -> tpcds_partitioned_parquet_snap.item.i_category, RF008[bloom] -> tpcds_partitioned_parquet_snap.item.i_category
| stored statistics:
| table: rows=360.00K size=33.54MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=101.54K
-| parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.item.i_category)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.item.i_current_price IS NOT NULL, tpcds_partitioned_parquet_snap.item.i_category IS NOT NULL, tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=11 row-size=26B cardinality=36.00K cost=82199
-| in pipelines: 11(GETNEXT)
-|
-10:HASH JOIN [INNER JOIN, PARTITIONED]
-| hash-table-id=02
-| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=0,2,6,9,7 row-size=50B cardinality=130.45M cost=60086864
-| in pipelines: 00(GETNEXT), 08(OPEN)
-|
-|--F16:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=20 (adjusted from 120)
-| | Per-Instance Resources: mem-estimate=101.93MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
-| | max-parallelism=20 segment-costs=[38025000] cpu-comparison-result=40 [max(40 (self) vs 20 (sum children))]
-| JOIN BUILD
-| | join-table-id=02 plan-id=03 cohort-id=01
-| | build expressions: tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| | runtime filters: RF002[bloom] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk, RF003[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| | mem-estimate=75.59MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
-| |
-| 26:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_sk)]
-| | mem-estimate=10.33MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=9,7 row-size=26B cardinality=30.00M cost=8025000
-| | in pipelines: 08(GETNEXT)
-| |
-| F07:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_address.ca_address_sk)] hosts=10 instances=10
-| Per-Instance Resources: mem-estimate=12.87MB mem-reservation=0B thread-reservation=1
-| max-parallelism=10 segment-costs=[93336000]
-| 09:HASH JOIN [INNER JOIN, PARTITIONED]
-| | hash-table-id=03
-| | hash predicates: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk = tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk = tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
-| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=9,7 row-size=26B cardinality=30.00M cost=9283500
-| | in pipelines: 08(GETNEXT), 07(OPEN)
-| |
-| |--F17:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_address.ca_address_sk)] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=151.22MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
-| | | max-parallelism=10 segment-costs=[34437000]
-| | JOIN BUILD
-| | | join-table-id=03 plan-id=04 cohort-id=03
-| | | build expressions: tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
-| | | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.customer.c_current_addr_sk, RF009[min_max] <- tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
-| | | mem-estimate=125.11MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
-| | |
-| | 24:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_current_addr_sk)]
-| | | mem-estimate=10.12MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=7 row-size=8B cardinality=30.00M cost=4437000
-| | | in pipelines: 07(GETNEXT)
-| | |
-| | F06:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Instance Resources: mem-estimate=16.47MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=10 segment-costs=[34368000]
-| | 07:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
-| | HDFS partitions=1/1 files=1 size=1.55GB
-| | stored statistics:
-| | table: rows=30.00M size=1.55GB
-| | columns: all
-| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=7 row-size=8B cardinality=30.00M cost=3456000
-| | in pipelines: 07(GETNEXT)
-| |
-| 23:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_address.ca_address_sk)]
-| | mem-estimate=10.21MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=9 row-size=18B cardinality=15.00M cost=2908500
-| | in pipelines: 08(GETNEXT)
-| |
-| F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| Per-Host Shared Resources: mem-estimate=16.00MB mem-reservation=16.00MB thread-reservation=0 runtime-filters-memory=16.00MB
-| Per-Instance Resources: mem-estimate=16.86MB mem-reservation=128.00KB thread-reservation=1
-| max-parallelism=10 segment-costs=[26412000]
-| 08:SCAN HDFS [tpcds_partitioned_parquet_snap.customer_address, RANDOM]
-| HDFS partitions=1/1 files=1 size=307.36MB
-| runtime filters: RF009[min_max] -> tpcds_partitioned_parquet_snap.customer_address.ca_address_sk, RF008[bloom] -> tpcds_partitioned_parquet_snap.customer_address.ca_address_sk
-| stored statistics:
-| table: rows=15.00M size=307.36MB
-| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=1.58M
-| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=9 row-size=18B cardinality=15.00M cost=1296000
-| in pipelines: 08(GETNEXT)
+| tuple-ids=7 row-size=26B cardinality=358.67K cost=102431
+| in pipelines: 07(GETNEXT)
|
-25:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)]
-| mem-estimate=14.22MB mem-reservation=0B thread-reservation=0
-| tuple-ids=0,2,6 row-size=24B cardinality=142.10M cost=41934948
-| in pipelines: 00(GETNEXT)
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
-Per-Host Shared Resources: mem-estimate=18.00MB mem-reservation=18.00MB thread-reservation=0 runtime-filters-memory=18.00MB
-Per-Instance Resources: mem-estimate=18.81MB mem-reservation=8.00MB thread-reservation=1
-max-parallelism=400 segment-costs=[3992361786]
06:HASH JOIN [INNER JOIN, BROADCAST]
| hash-table-id=04
| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=0,2,6 row-size=24B cardinality=142.10M cost=2060108489
+| tuple-ids=0,2,6 row-size=24B cardinality=129.58M cost=1965957540
| in pipelines: 00(GETNEXT), 01(OPEN)
|
|--F18:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -459,7 +472,7 @@ max-parallelism=400 segment-costs=[3992361786]
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
| Per-Instance Resources: mem-estimate=16.08MB mem-reservation=512.00KB thread-reservation=1
-| max-parallelism=1 segment-costs=[8430]
+| max-parallelism=1 segment-costs=[12535]
| 05:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=05
| | hash predicates: tpcds_partitioned_parquet_snap.date_dim.d_month_seq = tpcds_partitioned_parquet_snap.date_dim_0.d_month_seq
@@ -537,29 +550,34 @@ max-parallelism=400 segment-costs=[3992361786]
| |
| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| HDFS partitions=1/1 files=1 size=2.17MB
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim.d_month_seq IS NOT NULL
| runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_month_seq, RF006[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_month_seq
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim.d_month_seq IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=2 row-size=8B cardinality=30(filtered from 73.05K) cost=8415
+| tuple-ids=2 row-size=8B cardinality=30(filtered from 73.05K) cost=12520
| in pipelines: 01(GETNEXT)
|
00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
- runtime filters: RF003[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
+ runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF003[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=0 row-size=12B cardinality=7.93G(filtered from 8.64G) cost=1492980801
+ extrapolated-rows=disabled max-scan-range-rows=12.22M
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=0 row-size=12B cardinality=7.57G(filtered from 7.88G) cost=1889184464
in pipelines: 00(GETNEXT)
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=401.81MB Threads=33
-Per-Host Resource Estimates: Memory=942MB
+Max Per-Host Resource Reservation: Memory=651.81MB Threads=33
+Per-Host Resource Estimates: Memory=1.32GB
F13:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=4.02MB mem-reservation=4.00MB thread-reservation=1
| max-parallelism=1 segment-costs=[11] cpu-comparison-result=140 [max(1 (self) vs 140 (sum children))]
@@ -596,210 +614,214 @@ max-parallelism=10 segment-costs=[3222, 6, 4] cpu-comparison-result=140 [max(10
| tuple-ids=17 row-size=22B cardinality=6.12K cost=1299
| in pipelines: 00(GETNEXT)
|
-F08:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=20 (adjusted from 120)
-Per-Instance Resources: mem-estimate=25.23MB mem-reservation=2.00MB thread-reservation=1
-max-parallelism=20 segment-costs=[197259856, 11823] cpu-comparison-result=140 [max(140 (self) vs 65 (sum children))]
+F11:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=20 (adjusted from 120)
+Per-Instance Resources: mem-estimate=32.24MB mem-reservation=2.00MB thread-reservation=1
+max-parallelism=20 segment-costs=[168174460, 11823] cpu-comparison-result=140 [max(140 (self) vs 65 (sum children))]
16:AGGREGATE [STREAMING]
| output: count()
| group by: tpcds_partitioned_parquet_snap.customer_address.ca_state
| mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=17 row-size=22B cardinality=6.12K cost=38152863
+| tuple-ids=17 row-size=22B cardinality=6.12K cost=34791653
| in pipelines: 00(GETNEXT)
|
-15:HASH JOIN [INNER JOIN, BROADCAST]
+15:HASH JOIN [INNER JOIN, PARTITIONED]
| hash-table-id=00
+| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=0,2,6,7,12,15,13 row-size=102B cardinality=118.89M cost=54763705
+| in pipelines: 00(GETNEXT), 13(OPEN)
+|
+|--F14:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=20 (adjusted from 120)
+| | Per-Instance Resources: mem-estimate=101.93MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | max-parallelism=20 segment-costs=[38025000] cpu-comparison-result=40 [max(40 (self) vs 20 (sum children))]
+| JOIN BUILD
+| | join-table-id=00 plan-id=01 cohort-id=01
+| | build expressions: tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk, RF001[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk
+| | mem-estimate=75.59MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| |
+| 30:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_sk)]
+| | mem-estimate=10.33MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=15,13 row-size=26B cardinality=30.00M cost=8025000
+| | in pipelines: 13(GETNEXT)
+| |
+| F10:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_address.ca_address_sk)] hosts=10 instances=10
+| Per-Instance Resources: mem-estimate=12.87MB mem-reservation=0B thread-reservation=1
+| max-parallelism=10 segment-costs=[93336000]
+| 14:HASH JOIN [INNER JOIN, PARTITIONED]
+| | hash-table-id=01
+| | hash predicates: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk = tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
+| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk = tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
+| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
+| | tuple-ids=15,13 row-size=26B cardinality=30.00M cost=9283500
+| | in pipelines: 13(GETNEXT), 12(OPEN)
+| |
+| |--F15:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_address.ca_address_sk)] hosts=10 instances=10
+| | | Per-Instance Resources: mem-estimate=151.22MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
+| | | max-parallelism=10 segment-costs=[34437000]
+| | JOIN BUILD
+| | | join-table-id=01 plan-id=02 cohort-id=02
+| | | build expressions: tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
+| | | runtime filters: RF010[bloom] <- tpcds_partitioned_parquet_snap.customer.c_current_addr_sk, RF011[min_max] <- tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
+| | | mem-estimate=125.11MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
+| | |
+| | 28:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_current_addr_sk)]
+| | | mem-estimate=10.12MB mem-reservation=0B thread-reservation=0
+| | | tuple-ids=13 row-size=8B cardinality=30.00M cost=4437000
+| | | in pipelines: 12(GETNEXT)
+| | |
+| | F09:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=16.47MB mem-reservation=1.00MB thread-reservation=1
+| | max-parallelism=10 segment-costs=[36054000]
+| | 12:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
+| | HDFS partitions=1/1 files=1 size=1.55GB
+| | predicates: tpcds_partitioned_parquet_snap.customer.c_current_addr_sk IS NOT NULL, tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
+| | stored statistics:
+| | table: rows=30.00M size=1.55GB
+| | columns: all
+| | extrapolated-rows=disabled max-scan-range-rows=3.10M
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer.c_current_addr_sk IS NOT NULL, tpcds_partitioned_parquet_snap.customer.c_customer_sk IS NOT NULL
+| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
+| | tuple-ids=13 row-size=8B cardinality=30.00M cost=5142000
+| | in pipelines: 12(GETNEXT)
+| |
+| 27:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_address.ca_address_sk)]
+| | mem-estimate=10.21MB mem-reservation=0B thread-reservation=0
+| | tuple-ids=15 row-size=18B cardinality=15.00M cost=2908500
+| | in pipelines: 13(GETNEXT)
+| |
+| F08:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| Per-Host Shared Resources: mem-estimate=16.00MB mem-reservation=16.00MB thread-reservation=0 runtime-filters-memory=16.00MB
+| Per-Instance Resources: mem-estimate=16.86MB mem-reservation=128.00KB thread-reservation=1
+| max-parallelism=10 segment-costs=[26833500]
+| 13:SCAN HDFS [tpcds_partitioned_parquet_snap.customer_address, RANDOM]
+| HDFS partitions=1/1 files=1 size=307.36MB
+| predicates: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk IS NOT NULL
+| runtime filters: RF011[min_max] -> tpcds_partitioned_parquet_snap.customer_address.ca_address_sk, RF010[bloom] -> tpcds_partitioned_parquet_snap.customer_address.ca_address_sk
+| stored statistics:
+| table: rows=15.00M size=307.36MB
+| columns: all
+| extrapolated-rows=disabled max-scan-range-rows=1.58M
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk IS NOT NULL
+| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
+| tuple-ids=15 row-size=18B cardinality=15.00M cost=1717500
+| in pipelines: 13(GETNEXT)
+|
+29:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)]
+| mem-estimate=21.23MB mem-reservation=0B thread-reservation=0
+| tuple-ids=0,2,6,7,12 row-size=76B cardinality=129.52M cost=78619102
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
+Per-Host Shared Resources: mem-estimate=18.00MB mem-reservation=18.00MB thread-reservation=0 runtime-filters-memory=18.00MB
+Per-Instance Resources: mem-estimate=23.49MB mem-reservation=512.00KB thread-reservation=1
+max-parallelism=490 segment-costs=[4877793238]
+11:HASH JOIN [INNER JOIN, BROADCAST]
+| hash-table-id=02
| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk = tpcds_partitioned_parquet_snap.item.i_item_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_item_sk = tpcds_partitioned_parquet_snap.item.i_item_sk
| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=0,2,6,9,7,11,16 row-size=102B cardinality=130.38M cost=57085181
-| in pipelines: 00(GETNEXT), 11(OPEN)
+| tuple-ids=0,2,6,7,12 row-size=76B cardinality=129.52M cost=56705341
+| in pipelines: 00(GETNEXT), 07(OPEN)
|
-|--F14:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=10
-| | Per-Instance Resources: mem-estimate=79.23MB mem-reservation=69.00MB thread-reservation=1 runtime-filters-memory=1.00MB
+|--F16:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
+| | Per-Instance Resources: mem-estimate=419.23MB mem-reservation=409.00MB thread-reservation=1 runtime-filters-memory=1.00MB
| | max-parallelism=10 segment-costs=[838440] cpu-comparison-result=14 [max(14 (self) vs 4 (sum children))]
| JOIN BUILD
-| | join-table-id=00 plan-id=01 cohort-id=01
+| | join-table-id=02 plan-id=03 cohort-id=01
| | build expressions: tpcds_partitioned_parquet_snap.item.i_item_sk
-| | runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.item.i_item_sk, RF001[min_max] <- tpcds_partitioned_parquet_snap.item.i_item_sk
-| | mem-estimate=68.00MB mem-reservation=68.00MB spill-buffer=2.00MB thread-reservation=0 cost=360000
+| | runtime filters: RF002[bloom] <- tpcds_partitioned_parquet_snap.item.i_item_sk, RF003[min_max] <- tpcds_partitioned_parquet_snap.item.i_item_sk
+| | mem-estimate=408.00MB mem-reservation=408.00MB spill-buffer=2.00MB thread-reservation=0 cost=360000
| |
-| 30:EXCHANGE [BROADCAST]
+| 26:EXCHANGE [BROADCAST]
| | mem-estimate=10.23MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=11,16 row-size=52B cardinality=360.00K cost=478440
-| | in pipelines: 11(GETNEXT)
+| | tuple-ids=7,12 row-size=52B cardinality=360.00K cost=478440
+| | in pipelines: 07(GETNEXT)
| |
-| F09:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+| F05:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
| Per-Instance Resources: mem-estimate=16.23MB mem-reservation=512.00KB thread-reservation=1
-| max-parallelism=4 segment-costs=[222576]
-| 14:HASH JOIN [INNER JOIN, BROADCAST]
-| | hash-table-id=01
+| max-parallelism=4 segment-costs=[325572]
+| 10:HASH JOIN [INNER JOIN, BROADCAST]
+| | hash-table-id=03
| | hash predicates: tpcds_partitioned_parquet_snap.item.i_category = tpcds_partitioned_parquet_snap.item_0.i_category
| | fk/pk conjuncts: none
| | other join predicates: tpcds_partitioned_parquet_snap.item.i_current_price > multiply(CAST(1.2 AS DECIMAL(2,1)), avg(tpcds_partitioned_parquet_snap.item_0.i_current_price))
| | mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| | tuple-ids=11,16 row-size=52B cardinality=360.00K cost=74466
-| | in pipelines: 11(GETNEXT), 28(OPEN)
+| | tuple-ids=7,12 row-size=52B cardinality=360.00K cost=157230
+| | in pipelines: 07(GETNEXT), 24(OPEN)
| |
-| |--F15:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+| |--F17:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
| | | Per-Instance Resources: mem-estimate=2.95MB mem-reservation=2.94MB thread-reservation=1 runtime-filters-memory=1.00MB
| | | max-parallelism=4 segment-costs=[14] cpu-comparison-result=4 [max(4 (self) vs 4 (sum children))]
| | JOIN BUILD
-| | | join-table-id=01 plan-id=02 cohort-id=02
+| | | join-table-id=03 plan-id=04 cohort-id=03
| | | build expressions: tpcds_partitioned_parquet_snap.item_0.i_category
-| | | runtime filters: RF010[bloom] <- tpcds_partitioned_parquet_snap.item_0.i_category, RF011[min_max] <- tpcds_partitioned_parquet_snap.item_0.i_category
+| | | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.item_0.i_category, RF009[min_max] <- tpcds_partitioned_parquet_snap.item_0.i_category
| | | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0 cost=10
| | |
-| | 29:EXCHANGE [BROADCAST]
+| | 25:EXCHANGE [BROADCAST]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=16 row-size=26B cardinality=10 cost=4
-| | | in pipelines: 28(GETNEXT)
+| | | tuple-ids=12 row-size=26B cardinality=10 cost=4
+| | | in pipelines: 24(GETNEXT)
| | |
-| | F11:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.item_0.i_category)] hosts=4 instances=4
+| | F07:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.item_0.i_category)] hosts=4 instances=4
| | Per-Instance Resources: mem-estimate=10.12MB mem-reservation=1.94MB thread-reservation=1
| | max-parallelism=4 segment-costs=[46, 0] cpu-comparison-result=4 [max(4 (self) vs 4 (sum children))]
-| | 28:AGGREGATE [FINALIZE]
+| | 24:AGGREGATE [FINALIZE]
| | | output: avg:merge(tpcds_partitioned_parquet_snap.item_0.i_current_price)
| | | group by: tpcds_partitioned_parquet_snap.item_0.i_category
| | | mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| | | tuple-ids=16 row-size=26B cardinality=10 cost=37
-| | | in pipelines: 28(GETNEXT), 12(OPEN)
+| | | tuple-ids=12 row-size=26B cardinality=10 cost=37
+| | | in pipelines: 24(GETNEXT), 08(OPEN)
| | |
-| | 27:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.item_0.i_category)]
+| | 23:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.item_0.i_category)]
| | | mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=15 row-size=26B cardinality=40 cost=9
-| | | in pipelines: 12(GETNEXT)
+| | | tuple-ids=11 row-size=26B cardinality=40 cost=9
+| | | in pipelines: 08(GETNEXT)
| | |
-| | F10:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
+| | F06:PLAN FRAGMENT [RANDOM] hosts=4 instances=4
| | Per-Instance Resources: mem-estimate=26.47MB mem-reservation=2.50MB thread-reservation=1
-| | max-parallelism=4 segment-costs=[72097, 87]
-| | 13:AGGREGATE [STREAMING]
+| | max-parallelism=4 segment-costs=[166617, 87]
+| | 09:AGGREGATE [STREAMING]
| | | output: avg(tpcds_partitioned_parquet_snap.item_0.i_current_price)
| | | group by: tpcds_partitioned_parquet_snap.item_0.i_category
| | | mem-estimate=10.00MB mem-reservation=2.00MB spill-buffer=64.00KB thread-reservation=0
-| | | tuple-ids=15 row-size=26B cardinality=40 cost=10634
-| | | in pipelines: 12(GETNEXT)
+| | | tuple-ids=11 row-size=26B cardinality=40 cost=105154
+| | | in pipelines: 08(GETNEXT)
| | |
-| | 12:SCAN HDFS [tpcds_partitioned_parquet_snap.item, RANDOM]
+| | 08:SCAN HDFS [tpcds_partitioned_parquet_snap.item, RANDOM]
| | HDFS partitions=1/1 files=1 size=33.54MB
-| | predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.item_0.i_category)
+| | predicates: tpcds_partitioned_parquet_snap.item_0.i_category IS NOT NULL
| | stored statistics:
| | table: rows=360.00K size=33.54MB
| | columns: all
| | extrapolated-rows=disabled max-scan-range-rows=101.54K
-| | parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.item_0.i_category)
+| | parquet dictionary predicates: tpcds_partitioned_parquet_snap.item_0.i_category IS NOT NULL
| | mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| | tuple-ids=13 row-size=22B cardinality=36.00K cost=61463
-| | in pipelines: 12(GETNEXT)
+| | tuple-ids=9 row-size=22B cardinality=359.14K cost=61463
+| | in pipelines: 08(GETNEXT)
| |
-| 11:SCAN HDFS [tpcds_partitioned_parquet_snap.item, RANDOM]
+| 07:SCAN HDFS [tpcds_partitioned_parquet_snap.item, RANDOM]
| HDFS partitions=1/1 files=1 size=33.54MB
-| predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.item.i_category)
-| runtime filters: RF011[min_max] -> tpcds_partitioned_parquet_snap.item.i_category, RF010[bloom] -> tpcds_partitioned_parquet_snap.item.i_category
+| predicates: tpcds_partitioned_parquet_snap.item.i_current_price IS NOT NULL, tpcds_partitioned_parquet_snap.item.i_category IS NOT NULL, tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
+| runtime filters: RF009[min_max] -> tpcds_partitioned_parquet_snap.item.i_category, RF008[bloom] -> tpcds_partitioned_parquet_snap.item.i_category
| stored statistics:
| table: rows=360.00K size=33.54MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=101.54K
-| parquet dictionary predicates: is_not_null_pred(tpcds_partitioned_parquet_snap.item.i_category)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.item.i_current_price IS NOT NULL, tpcds_partitioned_parquet_snap.item.i_category IS NOT NULL, tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=11 row-size=26B cardinality=36.00K cost=82199
-| in pipelines: 11(GETNEXT)
-|
-10:HASH JOIN [INNER JOIN, PARTITIONED]
-| hash-table-id=02
-| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk = tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=0,2,6,9,7 row-size=50B cardinality=130.45M cost=60086864
-| in pipelines: 00(GETNEXT), 08(OPEN)
-|
-|--F16:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)] hosts=10 instances=20 (adjusted from 120)
-| | Per-Instance Resources: mem-estimate=101.93MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
-| | max-parallelism=20 segment-costs=[38025000] cpu-comparison-result=40 [max(40 (self) vs 20 (sum children))]
-| JOIN BUILD
-| | join-table-id=02 plan-id=03 cohort-id=01
-| | build expressions: tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| | runtime filters: RF002[bloom] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk, RF003[min_max] <- tpcds_partitioned_parquet_snap.customer.c_customer_sk
-| | mem-estimate=75.59MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
-| |
-| 26:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_customer_sk)]
-| | mem-estimate=10.33MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=9,7 row-size=26B cardinality=30.00M cost=8025000
-| | in pipelines: 08(GETNEXT)
-| |
-| F07:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_address.ca_address_sk)] hosts=10 instances=10
-| Per-Instance Resources: mem-estimate=12.87MB mem-reservation=0B thread-reservation=1
-| max-parallelism=10 segment-costs=[93336000]
-| 09:HASH JOIN [INNER JOIN, PARTITIONED]
-| | hash-table-id=03
-| | hash predicates: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk = tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
-| | fk/pk conjuncts: tpcds_partitioned_parquet_snap.customer_address.ca_address_sk = tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
-| | mem-estimate=0B mem-reservation=0B spill-buffer=2.00MB thread-reservation=0
-| | tuple-ids=9,7 row-size=26B cardinality=30.00M cost=9283500
-| | in pipelines: 08(GETNEXT), 07(OPEN)
-| |
-| |--F17:PLAN FRAGMENT [HASH(tpcds_partitioned_parquet_snap.customer_address.ca_address_sk)] hosts=10 instances=10
-| | | Per-Instance Resources: mem-estimate=151.22MB mem-reservation=50.00MB thread-reservation=1 runtime-filters-memory=16.00MB
-| | | max-parallelism=10 segment-costs=[34437000]
-| | JOIN BUILD
-| | | join-table-id=03 plan-id=04 cohort-id=03
-| | | build expressions: tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
-| | | runtime filters: RF008[bloom] <- tpcds_partitioned_parquet_snap.customer.c_current_addr_sk, RF009[min_max] <- tpcds_partitioned_parquet_snap.customer.c_current_addr_sk
-| | | mem-estimate=125.11MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0 cost=30000000
-| | |
-| | 24:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer.c_current_addr_sk)]
-| | | mem-estimate=10.12MB mem-reservation=0B thread-reservation=0
-| | | tuple-ids=7 row-size=8B cardinality=30.00M cost=4437000
-| | | in pipelines: 07(GETNEXT)
-| | |
-| | F06:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| | Per-Instance Resources: mem-estimate=16.47MB mem-reservation=1.00MB thread-reservation=1
-| | max-parallelism=10 segment-costs=[34368000]
-| | 07:SCAN HDFS [tpcds_partitioned_parquet_snap.customer, RANDOM]
-| | HDFS partitions=1/1 files=1 size=1.55GB
-| | stored statistics:
-| | table: rows=30.00M size=1.55GB
-| | columns: all
-| | extrapolated-rows=disabled max-scan-range-rows=3.10M
-| | mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| | tuple-ids=7 row-size=8B cardinality=30.00M cost=3456000
-| | in pipelines: 07(GETNEXT)
-| |
-| 23:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.customer_address.ca_address_sk)]
-| | mem-estimate=10.21MB mem-reservation=0B thread-reservation=0
-| | tuple-ids=9 row-size=18B cardinality=15.00M cost=2908500
-| | in pipelines: 08(GETNEXT)
-| |
-| F05:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
-| Per-Host Shared Resources: mem-estimate=16.00MB mem-reservation=16.00MB thread-reservation=0 runtime-filters-memory=16.00MB
-| Per-Instance Resources: mem-estimate=16.86MB mem-reservation=128.00KB thread-reservation=1
-| max-parallelism=10 segment-costs=[26412000]
-| 08:SCAN HDFS [tpcds_partitioned_parquet_snap.customer_address, RANDOM]
-| HDFS partitions=1/1 files=1 size=307.36MB
-| runtime filters: RF009[min_max] -> tpcds_partitioned_parquet_snap.customer_address.ca_address_sk, RF008[bloom] -> tpcds_partitioned_parquet_snap.customer_address.ca_address_sk
-| stored statistics:
-| table: rows=15.00M size=307.36MB
-| columns: all
-| extrapolated-rows=disabled max-scan-range-rows=1.58M
-| mem-estimate=16.00MB mem-reservation=128.00KB thread-reservation=0
-| tuple-ids=9 row-size=18B cardinality=15.00M cost=1296000
-| in pipelines: 08(GETNEXT)
+| tuple-ids=7 row-size=26B cardinality=358.67K cost=102431
+| in pipelines: 07(GETNEXT)
|
-25:EXCHANGE [HASH(tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk)]
-| mem-estimate=14.22MB mem-reservation=0B thread-reservation=0
-| tuple-ids=0,2,6 row-size=24B cardinality=142.10M cost=41934948
-| in pipelines: 00(GETNEXT)
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=10 instances=120
-Per-Host Shared Resources: mem-estimate=18.00MB mem-reservation=18.00MB thread-reservation=0 runtime-filters-memory=18.00MB
-Per-Instance Resources: mem-estimate=18.81MB mem-reservation=8.00MB thread-reservation=1
-max-parallelism=400 segment-costs=[3992361786]
06:HASH JOIN [INNER JOIN, BROADCAST]
| hash-table-id=04
| hash predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| mem-estimate=0B mem-reservation=0B spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=0,2,6 row-size=24B cardinality=142.10M cost=2060108489
+| tuple-ids=0,2,6 row-size=24B cardinality=129.58M cost=1965957540
| in pipelines: 00(GETNEXT), 01(OPEN)
|
|--F18:PLAN FRAGMENT [RANDOM] hosts=10 instances=10
@@ -819,7 +841,7 @@ max-parallelism=400 segment-costs=[3992361786]
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
| Per-Host Shared Resources: mem-estimate=1.00MB mem-reservation=1.00MB thread-reservation=0 runtime-filters-memory=1.00MB
| Per-Instance Resources: mem-estimate=16.08MB mem-reservation=512.00KB thread-reservation=1
-| max-parallelism=1 segment-costs=[8430]
+| max-parallelism=1 segment-costs=[12535]
| 05:HASH JOIN [INNER JOIN, BROADCAST]
| | hash-table-id=05
| | hash predicates: tpcds_partitioned_parquet_snap.date_dim.d_month_seq = tpcds_partitioned_parquet_snap.date_dim_0.d_month_seq
@@ -897,24 +919,29 @@ max-parallelism=400 segment-costs=[3992361786]
| |
| 01:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim, RANDOM]
| HDFS partitions=1/1 files=1 size=2.17MB
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim.d_month_seq IS NOT NULL
| runtime filters: RF007[min_max] -> tpcds_partitioned_parquet_snap.date_dim.d_month_seq, RF006[bloom] -> tpcds_partitioned_parquet_snap.date_dim.d_month_seq
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL, tpcds_partitioned_parquet_snap.date_dim.d_month_seq IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=2 row-size=8B cardinality=30(filtered from 73.05K) cost=8415
+| tuple-ids=2 row-size=8B cardinality=30(filtered from 73.05K) cost=12520
| in pipelines: 01(GETNEXT)
|
00:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales, RANDOM]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
- runtime filters: RF003[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
+ runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF003[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=0 row-size=12B cardinality=7.93G(filtered from 8.64G) cost=1492980801
+ extrapolated-rows=disabled max-scan-range-rows=12.22M
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_customer_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=0 row-size=12B cardinality=7.57G(filtered from 7.88G) cost=1889184464
in pipelines: 00(GETNEXT)
====
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q07.test b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q07.test
index 26bcb5b33..6ef8ea27a 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q07.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/calcite_tpcds/tpcds-q07.test
@@ -21,11 +21,11 @@ select i_item_id,
# end query 7 in stream 0 using template query7.tpl
---- PLAN
-Max Per-Host Resource Reservation: Memory=75.38MB Threads=1
+Max Per-Host Resource Reservation: Memory=67.88MB Threads=1
Per-Host Resource Estimates: Memory=130MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Instance Resources: mem-estimate=130.38MB mem-reservation=75.38MB thread-reservation=1 runtime-filters-memory=4.00MB
-| max-parallelism=1 segment-costs=[2627109148, 1831104, 500]
+| Per-Instance Resources: mem-estimate=130.38MB mem-reservation=67.88MB thread-reservation=1 runtime-filters-memory=4.00MB
+| max-parallelism=1 segment-costs=[2539730583, 1831104, 500]
PLAN-ROOT SINK
| output exprs: tpcds_partitioned_parquet_snap.item.i_item_id, avg(tpcds_partitioned_parquet_snap.store_sales.ss_quantity), avg(tpcds_partitioned_parquet_snap.store_sales.ss_list_price), avg(tpcds_partitioned_parquet_snap.store_sales.ss_coupon_amt), avg(tpcds_partitioned_parquet_snap.store_sales.ss_sales_price)
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0 cost=500
@@ -40,7 +40,7 @@ PLAN-ROOT SINK
| output: avg(CAST(tpcds_partitioned_parquet_snap.store_sales.ss_quantity AS BIGINT)), avg(tpcds_partitioned_parquet_snap.store_sales.ss_list_price), avg(tpcds_partitioned_parquet_snap.store_sales.ss_coupon_amt), avg(tpcds_partitioned_parquet_snap.store_sales.ss_sales_price)
| group by: tpcds_partitioned_parquet_snap.item.i_item_id
| mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB thread-reservation=0
-| tuple-ids=11 row-size=60B cardinality=185.57K cost=44952326
+| tuple-ids=11 row-size=60B cardinality=185.57K cost=40110232
| in pipelines: 09(GETNEXT), 01(OPEN)
|
08:HASH JOIN [INNER JOIN]
@@ -48,17 +48,19 @@ PLAN-ROOT SINK
| fk/pk conjuncts: none
| runtime filters: RF000[bloom] <- tpcds_partitioned_parquet_snap.item.i_item_sk, RF001[min_max] <- tpcds_partitioned_parquet_snap.item.i_item_sk
| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
-| tuple-ids=2,4,6,8,0 row-size=154B cardinality=88.49M cost=39094059
+| tuple-ids=2,4,6,8,0 row-size=154B cardinality=78.86M cost=34876379
| in pipelines: 01(GETNEXT), 00(OPEN)
|
|--00:SCAN HDFS [tpcds_partitioned_parquet_snap.item]
| HDFS partitions=1/1 files=1 size=33.54MB
+| predicates: tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
| stored statistics:
| table: rows=360.00K size=33.54MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=101.54K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.item.i_item_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=0 row-size=32B cardinality=360.00K cost=103680
+| tuple-ids=0 row-size=32B cardinality=360.00K cost=113796
| in pipelines: 00(GETNEXT)
|
07:HASH JOIN [INNER JOIN]
@@ -66,20 +68,20 @@ PLAN-ROOT SINK
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk = tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| runtime filters: RF002[bloom] <- tpcds_partitioned_parquet_snap.date_dim.d_date_sk
| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=2,4,6,8 row-size=122B cardinality=88.49M cost=38734432
+| tuple-ids=2,4,6,8 row-size=122B cardinality=78.86M cost=35963197
| in pipelines: 01(GETNEXT), 06(OPEN)
|
|--06:SCAN HDFS [tpcds_partitioned_parquet_snap.date_dim]
| HDFS partitions=1/1 files=1 size=2.17MB
-| predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2001 AS INT)
+| predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2001 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| stored statistics:
| table: rows=73.05K size=2.17MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=73.05K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2001 AS INT)
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2001 AS INT)
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.date_dim.d_year = CAST(2001 AS INT), tpcds_partitioned_parquet_snap.date_dim.d_date_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
-| tuple-ids=8 row-size=8B cardinality=373 cost=10467
+| tuple-ids=8 row-size=8B cardinality=373 cost=12520
| in pipelines: 06(GETNEXT)
|
05:HASH JOIN [INNER JOIN]
@@ -87,20 +89,20 @@ PLAN-ROOT SINK
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk = tpcds_partitioned_parquet_snap.customer_demographics.cd_demo_sk
| runtime filters: RF004[bloom] <- tpcds_partitioned_parquet_snap.customer_demographics.cd_demo_sk, RF005[min_max] <- tpcds_partitioned_parquet_snap.customer_demographics.cd_demo_sk
| mem-estimate=8.50MB mem-reservation=8.50MB spill-buffer=512.00KB thread-reservation=0
-| tuple-ids=2,4,6 row-size=114B cardinality=88.49M(filtered from 432.75M) cost=101209716
+| tuple-ids=2,4,6 row-size=114B cardinality=84.50M(filtered from 385.62M) cost=91646227
| in pipelines: 01(GETNEXT), 04(OPEN)
|
|--04:SCAN HDFS [tpcds_partitioned_parquet_snap.customer_demographics]
| HDFS partitions=1/1 files=1 size=11.15MB
-| predicates: tpcds_partitioned_parquet_snap.customer_demographics.cd_education_status = 'College', tpcds_partitioned_parquet_snap.customer_demographics.cd_marital_status = 'W', tpcds_partitioned_parquet_snap.customer_demographics.cd_gender = 'F'
+| predicates: tpcds_partitioned_parquet_snap.customer_demographics.cd_education_status = 'College', tpcds_partitioned_parquet_snap.customer_demographics.cd_marital_status = 'W', tpcds_partitioned_parquet_snap.customer_demographics.cd_gender = 'F', tpcds_partitioned_parquet_snap.customer_demographics.cd_demo_sk IS NOT NULL
| stored statistics:
| table: rows=1.92M size=11.15MB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=197.97K
| parquet statistics predicates: tpcds_partitioned_parquet_snap.customer_demographics.cd_education_status = 'College', tpcds_partitioned_parquet_snap.customer_demographics.cd_marital_status = 'W', tpcds_partitioned_parquet_snap.customer_demographics.cd_gender = 'F'
-| parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_demographics.cd_education_status = 'College', tpcds_partitioned_parquet_snap.customer_demographics.cd_marital_status = 'W', tpcds_partitioned_parquet_snap.customer_demographics.cd_gender = 'F'
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.customer_demographics.cd_education_status = 'College', tpcds_partitioned_parquet_snap.customer_demographics.cd_marital_status = 'W', tpcds_partitioned_parquet_snap.customer_demographics.cd_gender = 'F', tpcds_partitioned_parquet_snap.customer_demographics.cd_demo_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=1.00MB thread-reservation=0
-| tuple-ids=6 row-size=52B cardinality=97.40K cost=592621
+| tuple-ids=6 row-size=52B cardinality=97.40K cost=646596
| in pipelines: 04(GETNEXT)
|
03:HASH JOIN [INNER JOIN]
@@ -108,33 +110,37 @@ PLAN-ROOT SINK
| fk/pk conjuncts: tpcds_partitioned_parquet_snap.store_sales.ss_promo_sk = tpcds_partitioned_parquet_snap.promotion.p_promo_sk
| runtime filters: RF006[bloom] <- tpcds_partitioned_parquet_snap.promotion.p_promo_sk, RF007[min_max] <- tpcds_partitioned_parquet_snap.promotion.p_promo_sk
| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB thread-reservation=0
-| tuple-ids=2,4 row-size=62B cardinality=88.49M(filtered from 8.64G) cost=1588256906
+| tuple-ids=2,4 row-size=62B cardinality=84.50M(filtered from 7.70G) cost=1416761167
| in pipelines: 01(GETNEXT), 02(OPEN)
|
|--02:SCAN HDFS [tpcds_partitioned_parquet_snap.promotion]
| HDFS partitions=1/1 files=1 size=100.50KB
-| predicates: tpcds_partitioned_parquet_snap.promotion.p_channel_email = 'N' OR tpcds_partitioned_parquet_snap.promotion.p_channel_event = 'N'
+| predicates: tpcds_partitioned_parquet_snap.promotion.p_channel_email = 'N' OR tpcds_partitioned_parquet_snap.promotion.p_channel_event = 'N', tpcds_partitioned_parquet_snap.promotion.p_promo_sk IS NOT NULL
| stored statistics:
| table: rows=1.80K size=100.50KB
| columns: all
| extrapolated-rows=disabled max-scan-range-rows=1.80K
+| parquet dictionary predicates: tpcds_partitioned_parquet_snap.promotion.p_promo_sk IS NOT NULL
| mem-estimate=16.00MB mem-reservation=32.00KB thread-reservation=0
-| tuple-ids=4 row-size=30B cardinality=1.80K cost=206
+| tuple-ids=4 row-size=30B cardinality=1.80K cost=256
| in pipelines: 02(GETNEXT)
|
01:SCAN HDFS [tpcds_partitioned_parquet_snap.store_sales]
- HDFS partitions=1824/1824 files=1824 size=389.90GB
+ partition predicates: tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk IS NOT NULL
+ HDFS partitions=1823/1824 files=1823 size=389.69GB
+ predicates: tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_sales.ss_promo_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
runtime filters: RF001[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF005[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk, RF007[min_max] -> tpcds_partitioned_parquet_snap.store_sales.ss_promo_sk, RF000[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_item_sk, RF002[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_sold_date_sk, RF004[bloom] -> tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk, RF006[bloom] -> tpcds_partitione [...]
stored statistics:
table: rows=8.64G size=389.90GB
- partitions: 1824/1824 rows=8.64G
+ partitions: 1823/1823 rows=8.25G
columns: all
- extrapolated-rows=disabled max-scan-range-rows=390.22M est-scan-range=374(filtered from 1824)
- mem-estimate=16.00MB mem-reservation=8.00MB thread-reservation=0
- tuple-ids=2 row-size=32B cardinality=88.49M(filtered from 8.64G) cost=814154735
+ extrapolated-rows=disabled max-scan-range-rows=12.22M est-scan-range=373(filtered from 1823)
+ parquet dictionary predicates: tpcds_partitioned_parquet_snap.store_sales.ss_cdemo_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_sales.ss_promo_sk IS NOT NULL, tpcds_partitioned_parquet_snap.store_sales.ss_item_sk IS NOT NULL
+ mem-estimate=16.00MB mem-reservation=512.00KB thread-reservation=0
+ tuple-ids=2 row-size=32B cardinality=84.50M(filtered from 7.70G) cost=919600213
in pipelines: 01(GETNEXT)
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=873.41MB Threads=22
+Max Per-Host Resource Reservation: Memory=783.41MB Threads=22
Per-Host Resource Estimates: Memory=1.08GB
F06:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=4.06MB mem-reservation=4.00MB thread-reservation=1
... 129563 lines suppressed ...