You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2026/10/21 07:54:42 UTC
(impala) 01/02: IMPALA-15175: Support HBO for AnalyticEvalNode cardinality
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 0207eaa0a2779b51a7c3878d455c10b92e9eb5c6
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Mon Jul 13 17:12:36 2026 +0800
IMPALA-15175: Support HBO for AnalyticEvalNode cardinality
AnalyticEvalNode appends analytic results to the input tuples as new
columns. Though they won't change the cardinality, the analytic results
could be used in downstream group-by keys, join keys, or predicates,
which matters in cardinalities of downstream nodes. This patch adds
AnalyticEvalNode in the HBO key strings so its parent nodes can still be
tracked in HBO. Note that cardinality of AnalyticEvalNode is still
untracked since it can be simply deduced from its input cardinality.
The HBO key string of an AnalyticEvalNode consists of
- Analytic functions
- PartitionBy Exprs
- OrderBy Exprs
- Window
- The child's HBO key string
Exprs are canonicalized with operand indexes if the child node has
multiple output tuples, e.g., JoinNode. PartitionBy Exprs are sorted
since the order doesn't matter. OrderBy Exprs are added using the
original order.
Testing
- Added FE tests and e2e tests
Assisted-by: Opus 4.8 (Claude Code)
Change-Id: I8d87ab472db9273622f48fa62bb1d8c12e04e86b
Reviewed-on: http://gerrit.cloudera.org:8080/24608
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../org/apache/impala/analysis/AnalyticExpr.java | 9 +-
.../apache/impala/planner/AnalyticEvalNode.java | 68 ++++++
.../apache/impala/planner/HboKeyStringTest.java | 217 +++++++++++++++++++
.../queries/QueryTest/hbo-analytic-agg.test | 67 ++++++
.../queries/QueryTest/hbo-analytic-join.test | 229 +++++++++++++++++++++
tests/query_test/test_hbo.py | 28 +++
6 files changed, 617 insertions(+), 1 deletion(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java b/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java
index 851f0bba39..2318ad1664 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java
@@ -154,7 +154,14 @@ public class AnalyticExpr extends Expr {
@Override
public String toSqlImpl(ToSqlOptions options) {
- if (sqlString_ != null) return sqlString_;
+ if (options.showForHbo()) {
+ // HBO keys must render fresh so operand qualifiers applied to child SlotRefs are
+ // picked up. syncWithChildren re-derives the mirror fields from children_ and is
+ // semantics-preserving (same convention as resetAnalysisState/substituteImpl).
+ syncWithChildren();
+ } else if (sqlString_ != null) {
+ return sqlString_;
+ }
StringBuilder sb = new StringBuilder();
sb.append(fnCall_.toSql(options)).append(" OVER (");
boolean needsSpace = false;
diff --git a/fe/src/main/java/org/apache/impala/planner/AnalyticEvalNode.java b/fe/src/main/java/org/apache/impala/planner/AnalyticEvalNode.java
index 2ab23a6433..e0fb2872d9 100644
--- a/fe/src/main/java/org/apache/impala/planner/AnalyticEvalNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/AnalyticEvalNode.java
@@ -18,7 +18,9 @@
package org.apache.impala.planner;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
import org.apache.impala.analysis.AnalyticExpr;
import org.apache.impala.analysis.AnalyticWindow;
@@ -42,7 +44,9 @@ import org.apache.impala.catalog.Function;
import org.apache.impala.common.Pair;
import org.apache.impala.thrift.TAnalyticNode;
import org.apache.impala.thrift.TExplainLevel;
+import org.apache.impala.thrift.THboStatsType;
import org.apache.impala.thrift.TPlanNode;
+import org.apache.impala.thrift.TPlanNodeRun;
import org.apache.impala.thrift.TPlanNodeType;
import org.apache.impala.thrift.TQueryOptions;
import org.apache.impala.util.ExprUtil;
@@ -132,6 +136,7 @@ public class AnalyticEvalNode extends PlanNode {
@Override
public void init(Analyzer analyzer) {
Preconditions.checkState(conjuncts_.isEmpty());
+ computeHboOperandQualifierMap(analyzer);
computeMemLayout(analyzer);
intermediateTupleDesc_.computeMemLayout();
@@ -309,6 +314,69 @@ public class AnalyticEvalNode extends PlanNode {
}
}
+ @Override
+ public boolean isOperandTransparent() { return true; }
+
+ @Override
+ public void appendScanInputStats(TPlanNodeRun execStats) {
+ getChild(0).appendScanInputStats(execStats);
+ }
+
+ /**
+ * Generates an HBO key string for this analytic node. Returns null if the child node
+ * doesn't support HBO.
+ */
+ @Override
+ public String generateHboKeyString(THboStatsType statsType,
+ CanonicalizationStrategy strategy) {
+ Preconditions.checkState(children_.size() == 1);
+ Preconditions.checkState(conjuncts_.isEmpty());
+ String childKey = getChild(0).generateHboKeyString(statsType, strategy);
+ if (childKey == null) {
+ LOG.debug("Not tracking {} in HBO since the child doesn't support HBO: {}",
+ getDisplayLabel(), getChild(0).getDisplayLabel());
+ return null;
+ }
+
+ StringBuilder sb = startHboKeyString(statsType, ":AnalyticEvalNode:");
+
+ // The analytic function calls, partition-by and order-by exprs all reference the
+ // input (the preceding sort output). When that input has multiple operands (a join
+ // below the analytic), their columns are qualified with the join's canonical operand
+ // index. A single-operand input (e.g. a scan) needs no qualification.
+ Map<TupleId, String> operandIdx = getChild(0).getHboOperandQualifierMap();
+
+ // Partition-by and function calls are order-independent and canonicalized into sorted
+ // sets.
+ List<String> fnStrs =
+ ExprCanonicalizer.canonicalizeExprs(analyticFnCalls_, strategy, operandIdx);
+ sb.append("|Fns:[").append(String.join(",", fnStrs)).append("]");
+ List<String> partStrs =
+ ExprCanonicalizer.canonicalizeExprs(partitionExprs_, strategy, operandIdx);
+ sb.append("|PartExprs:[").append(String.join(",", partStrs)).append("]");
+
+ // Order-by sequence is significant for ranking, so keep it as-is (do not sort). Each
+ // expr is qualified (via a singleton list) with the same operand index as above.
+ List<String> orderStrs = new ArrayList<>(orderByElements_.size());
+ for (OrderByElement e : orderByElements_) {
+ String exprStr = ExprCanonicalizer.canonicalizeExprs(
+ Collections.singletonList(e.getExpr()), strategy, operandIdx).get(0);
+ StringBuilder ob = new StringBuilder(exprStr);
+ ob.append(e.isAsc() ? " ASC" : " DESC")
+ .append(OrderByElement.nullsFirst(e.getNullsFirstParam(), e.isAsc()) ?
+ " NULLS FIRST" : " NULLS LAST");
+ orderStrs.add(ob.toString());
+ }
+ sb.append("|OrderExprs:[").append(String.join(",", orderStrs)).append("]");
+
+ sb.append("|Window:")
+ .append(analyticWindow_ == null ? "NONE" : analyticWindow_.toSql());
+ sb.append("|Child:[").append(childKey).append("]");
+ // Analytic predicates are evaluated by a SelectNode above it, so no predicates in
+ // the key.
+ return sb.toString();
+ }
+
@Override
protected String getNodeExplainString(String prefix, String detailPrefix,
TExplainLevel detailLevel) {
diff --git a/fe/src/test/java/org/apache/impala/planner/HboKeyStringTest.java b/fe/src/test/java/org/apache/impala/planner/HboKeyStringTest.java
index c02a39bdfa..63c8c2e5b0 100644
--- a/fe/src/test/java/org/apache/impala/planner/HboKeyStringTest.java
+++ b/fe/src/test/java/org/apache/impala/planner/HboKeyStringTest.java
@@ -1457,4 +1457,221 @@ public class HboKeyStringTest extends FrontendTestBase {
}
assertTopNStorePath(local, merge);
}
+
+ private AnalyticEvalNode singleAnalytic(String query) throws ImpalaException {
+ Map<Integer, PlanNode> nodes = collectPlanNodesInDistributedPlan(query);
+ int count = 0;
+ AnalyticEvalNode analytic = null;
+ for (PlanNode n : nodes.values()) {
+ if (n instanceof AnalyticEvalNode) {
+ analytic = (AnalyticEvalNode) n;
+ count++;
+ }
+ }
+ assertEquals("Expected exactly one AnalyticEvalNode for query: " + query, 1, count);
+ return analytic;
+ }
+
+ @Test
+ public void testAnalyticNodeKeys() throws ImpalaException {
+ String query = "select id, rank() over (partition by int_col order by id) "
+ + "from functional.alltypes where year = 2009";
+ AnalyticEvalNode analytic = singleAnalytic(query);
+
+ String prefix = "CARDINALITY:AnalyticEvalNode:|Fns:[rank()]|PartExprs:[int_col]|"
+ + "OrderExprs:[id ASC NULLS LAST]|"
+ + "Window:RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW|";
+ String expectedER = prefix
+ + "Child:[CARDINALITY:ScanNode:functional.alltypes|`year` = 2009]";
+ String expectedIPC = prefix
+ + "Child:[CARDINALITY:ScanNode:functional.alltypes|`year`=<CONST>]";
+
+ assertEquals(expectedER, analytic.generateHboKeyString(
+ THboStatsType.CARDINALITY, CanonicalizationStrategy.EXPR_REWRITE));
+ assertEquals(expectedIPC, analytic.generateHboKeyString(
+ THboStatsType.CARDINALITY,
+ CanonicalizationStrategy.IGNORE_PARTITION_CONSTANTS));
+ }
+
+ @Test
+ public void testAnalyticNodeLimit() throws ImpalaException {
+ // Same analytic as testAnalyticNodeKeys but with a query-level LIMIT, which lands on
+ // the topmost plan node (the AnalyticEvalNode).
+ String query = "select id, rank() over (partition by int_col order by id) "
+ + "from functional.alltypes where year = 2009 limit 5";
+ AnalyticEvalNode analytic = singleAnalytic(query);
+ assertTrue("Expected a valid LIMIT (> 0) on the AnalyticEvalNode",
+ analytic.getLimit() > 0);
+
+ // The LIMIT is also pushed into the analytic's underlying sort so it's a TOPN.
+ String childSortFmt = "CARDINALITY:SortNode:Type:TOPN|IncludeTies:false|"
+ + "OrderKind:LEXICAL|Limit:5|Offset:0|"
+ + "OrderExprs:[int_col ASC NULLS LAST,id ASC NULLS LAST]|"
+ + "Child:[CARDINALITY:ScanNode:functional.alltypes|%s]";
+ String prefix = "CARDINALITY:AnalyticEvalNode:|limit:5|Fns:[rank()]|"
+ + "PartExprs:[int_col]|OrderExprs:[id ASC NULLS LAST]|"
+ + "Window:RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW|Child:[%s]";
+ String expectedER =
+ String.format(prefix, String.format(childSortFmt, "`year` = 2009"));
+ String expectedIPC =
+ String.format(prefix, String.format(childSortFmt, "`year`=<CONST>"));
+
+ assertEquals(expectedER, analytic.generateHboKeyString(
+ THboStatsType.CARDINALITY, CanonicalizationStrategy.EXPR_REWRITE));
+ assertEquals(expectedIPC, analytic.generateHboKeyString(
+ THboStatsType.CARDINALITY,
+ CanonicalizationStrategy.IGNORE_PARTITION_CONSTANTS));
+ }
+
+ @Test
+ public void testAnalyticImplicitAndExplicitNullsMatch() throws ImpalaException {
+ // A plain "ORDER BY id" (ASC) and an explicit "ORDER BY id ASC NULLS LAST" are
+ // identical: NULLS LAST is the default for ASC.
+ String implicitQuery = "select id, rank() over (partition by int_col order by id) "
+ + "from functional.alltypes where year = 2009";
+ String explicitQuery = "select id, rank() over (partition by int_col order by id "
+ + "asc nulls last) from functional.alltypes where year = 2009";
+ AnalyticEvalNode implicitAnalytic = singleAnalytic(implicitQuery);
+ AnalyticEvalNode explicitAnalytic = singleAnalytic(explicitQuery);
+
+ String prefix = "CARDINALITY:AnalyticEvalNode:|Fns:[rank()]|PartExprs:[int_col]|"
+ + "OrderExprs:[id ASC NULLS LAST]|"
+ + "Window:RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW|";
+ String expectedER = prefix
+ + "Child:[CARDINALITY:ScanNode:functional.alltypes|`year` = 2009]";
+ String expectedIPC = prefix
+ + "Child:[CARDINALITY:ScanNode:functional.alltypes|`year`=<CONST>]";
+
+ // Both queries pin the same expected key under each canonicalization strategy.
+ assertEquals(expectedER, implicitAnalytic.generateHboKeyString(
+ THboStatsType.CARDINALITY, CanonicalizationStrategy.EXPR_REWRITE));
+ assertEquals(expectedER, explicitAnalytic.generateHboKeyString(
+ THboStatsType.CARDINALITY, CanonicalizationStrategy.EXPR_REWRITE));
+ assertEquals(expectedIPC, implicitAnalytic.generateHboKeyString(
+ THboStatsType.CARDINALITY,
+ CanonicalizationStrategy.IGNORE_PARTITION_CONSTANTS));
+ assertEquals(expectedIPC, explicitAnalytic.generateHboKeyString(
+ THboStatsType.CARDINALITY,
+ CanonicalizationStrategy.IGNORE_PARTITION_CONSTANTS));
+
+ for (CanonicalizationStrategy strategy : CanonicalizationStrategy.values()) {
+ assertEquals("Implicit ASC and explicit ASC NULLS LAST must produce the same "
+ + "AnalyticEvalNode key for " + strategy,
+ implicitAnalytic.generateHboKeyString(THboStatsType.CARDINALITY, strategy),
+ explicitAnalytic.generateHboKeyString(THboStatsType.CARDINALITY, strategy));
+ }
+ }
+
+ @Test
+ public void testAnalyticPartitionAndOrderAffectKey() throws ImpalaException {
+ // The partition-by and order-by exprs are part of the key, so analytic nodes that
+ // differ only in partitioning or ordering must hash differently.
+ String base = "select id, rank() over (partition by %s order by %s) "
+ + "from functional.alltypes where year = 2009";
+ AnalyticEvalNode byIntCol = singleAnalytic(String.format(base, "int_col", "id"));
+ AnalyticEvalNode byBigintCol =
+ singleAnalytic(String.format(base, "bigint_col", "id"));
+ AnalyticEvalNode orderDesc =
+ singleAnalytic(String.format(base, "int_col", "id desc"));
+
+ for (CanonicalizationStrategy strategy : CanonicalizationStrategy.values()) {
+ String kIntCol =
+ byIntCol.generateHboKeyString(THboStatsType.CARDINALITY, strategy);
+ String kBigintCol =
+ byBigintCol.generateHboKeyString(THboStatsType.CARDINALITY, strategy);
+ String kOrderDesc =
+ orderDesc.generateHboKeyString(THboStatsType.CARDINALITY, strategy);
+ assertNotEquals("Different PARTITION BY must hash differently for " + strategy,
+ kIntCol, kBigintCol);
+ assertNotEquals("Different ORDER BY must hash differently for " + strategy,
+ kIntCol, kOrderDesc);
+ }
+ }
+
+ @Test
+ public void testAnalyticOverJoin() throws ImpalaException {
+ // When an AnalyticEvalNode's input is a join, its partition-by / order-by (and
+ // function) columns reference several input operands and must be qualified with the
+ // join's canonical operand index. Swapping which operand supplies the partition vs.
+ // order column must therefore hash differently.
+ String q1 = "select a.id, rank() over (partition by a.int_col order by b.id) "
+ + "from functional.alltypes a join functional.alltypestiny b on a.id = b.id";
+ String q2 = "select a.id, rank() over (partition by b.int_col order by a.id) "
+ + "from functional.alltypes a join functional.alltypestiny b on a.id = b.id";
+ AnalyticEvalNode an1 = singleAnalytic(q1);
+ AnalyticEvalNode an2 = singleAnalytic(q2);
+
+ String childKey = "Window:RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW|"
+ + "Child:[CARDINALITY:JoinNode:INNER|Operands:[CARDINALITY:ScanNode:"
+ + "functional.alltypes,CARDINALITY:ScanNode:functional.alltypestiny]|"
+ + "Predicates:[op0.id=op1.id]]";
+ // q1: partition by a.int_col (alltypes=op0), order by b.id (alltypestiny=op1).
+ String expected1 = "CARDINALITY:AnalyticEvalNode:|Fns:[rank()]|PartExprs:"
+ + "[op0.int_col]|OrderExprs:[op1.id ASC NULLS LAST]|" + childKey;
+ // q2: partition by b.int_col (alltypestiny=op1), order by a.id (alltypes=op0).
+ String expected2 = "CARDINALITY:AnalyticEvalNode:|Fns:[rank()]|PartExprs:"
+ + "[op1.int_col]|OrderExprs:[op0.id ASC NULLS LAST]|" + childKey;
+
+ for (CanonicalizationStrategy strategy : CanonicalizationStrategy.values()) {
+ assertEquals(expected1,
+ an1.generateHboKeyString(THboStatsType.CARDINALITY, strategy));
+ assertEquals(expected2,
+ an2.generateHboKeyString(THboStatsType.CARDINALITY, strategy));
+ assertNotEquals("Swapped partition/order operands must hash differently for "
+ + strategy,
+ an1.generateHboKeyString(THboStatsType.CARDINALITY, strategy),
+ an2.generateHboKeyString(THboStatsType.CARDINALITY, strategy));
+ }
+ }
+
+ @Test
+ public void testAggOverAnalyticOverJoin() throws ImpalaException {
+ // Aggregation on top of an AnalyticEvalNode on top of a join, grouping on columns
+ // from different join operands and the analytic result.
+ String query = "select int_col, bigint_col, r, count(*) from ("
+ + "select a.int_col int_col, b.bigint_col bigint_col, "
+ + "rank() over (partition by a.int_col order by b.id) r "
+ + "from functional.alltypes a join functional.alltypestiny b on a.id = b.id"
+ + ") t group by int_col, bigint_col, r";
+
+ // The analytic's partition-by (a.int_col) and order-by (b.id) are qualified by the
+ // join operand index, just like the aggregation's grouping columns below.
+ String analyticChild = "CARDINALITY:AnalyticEvalNode:|Fns:[rank()]|"
+ + "PartExprs:[op0.int_col]|OrderExprs:[op1.id ASC NULLS LAST]|"
+ + "Window:RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW|"
+ + "Child:[CARDINALITY:JoinNode:INNER|Operands:[CARDINALITY:ScanNode:"
+ + "functional.alltypes,CARDINALITY:ScanNode:functional.alltypestiny]|"
+ + "Predicates:[op0.id=op1.id]]";
+ String expectedPreagg = "CARDINALITY:AggregationNode:FIRST|Preagg:true|"
+ + "GroupingSet:false|AggClasses:[0:Group:op0.int_col,op1.bigint_col,"
+ + "rank() OVER (PARTITION BY op0.int_col ORDER BY op1.id ASC "
+ + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)]"
+ + "|Child:[" + analyticChild + "]";
+
+ Map<Integer, PlanNode> nodes = collectPlanNodesInDistributedPlan(query);
+ PlanNode preagg = nodes.get(5);
+ assertEquals(expectedPreagg, preagg.generateHboKeyString(
+ THboStatsType.CARDINALITY, CanonicalizationStrategy.EXPR_REWRITE));
+ }
+
+ @Test
+ public void testAnalyticAsJoinOperand() throws ImpalaException {
+ // Test using the analytic result as the join key
+ String query = "select a.id from functional.alltypes a join ("
+ + "select id, rank() over (partition by int_col order by id) r "
+ + "from functional.alltypestiny) b on a.id = b.r";
+ JoinNode join = singleJoin(query);
+
+ String analyticOperand = "CARDINALITY:AnalyticEvalNode:|Fns:[rank()]|"
+ + "PartExprs:[int_col]|OrderExprs:[id ASC NULLS LAST]|"
+ + "Window:RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW|"
+ + "Child:[CARDINALITY:ScanNode:functional.alltypestiny]";
+ // The predicate (a.id = b.r) should be substituted and qualified to get rid of alias.
+ String expectedER = "CARDINALITY:JoinNode:INNER|Operands:["
+ + ALLTYPES_SCAN_KEY + "," + analyticOperand + "]|"
+ + "Predicates:[op0.id=rank() OVER (PARTITION BY op1.int_col ORDER BY op1.id ASC "
+ + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)]";
+ assertEquals(expectedER, join.generateHboKeyString(
+ THboStatsType.CARDINALITY, CanonicalizationStrategy.EXPR_REWRITE));
+ }
}
diff --git a/testdata/workloads/functional-query/queries/QueryTest/hbo-analytic-agg.test b/testdata/workloads/functional-query/queries/QueryTest/hbo-analytic-agg.test
new file mode 100644
index 0000000000..75535d3ff3
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/hbo-analytic-agg.test
@@ -0,0 +1,67 @@
+====
+---- QUERY
+EXPLAIN select count(*), rnk % 10 from
+ (select int_col, rank() over (order by id asc) as rnk
+ from functional.alltypes
+ where month=1 and string_col != 'test_agg_over_analytic') t
+group by rnk % 10
+---- PLAN
+PLAN-ROOT SINK
+|
+03:AGGREGATE [FINALIZE]
+| output: count(*)
+| group by: rank() % 10
+| row-size=16B cardinality=10 (from HBO)
+|
+02:ANALYTIC
+| functions: rank()
+| order by: id ASC
+| window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+| row-size=29B cardinality=620
+|
+04:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: id ASC
+|
+01:SORT
+| order by: id ASC
+| row-size=21B cardinality=620
+|
+00:SCAN HDFS [functional.alltypes]
+ partition predicates: `month` = 1
+ HDFS partitions=2/24 files=2 size=40.32KB
+ predicates: string_col != 'test_agg_over_analytic'
+ row-size=21B cardinality=620 (from HBO)
+====
+---- QUERY
+EXPLAIN select count(*), rnk % 10 from
+ (select int_col, rank() over (order by id asc) as rnk
+ from functional.alltypes
+ where month=2 and string_col != 'test_agg_over_analytic') t
+group by rnk % 10
+---- PLAN
+PLAN-ROOT SINK
+|
+03:AGGREGATE [FINALIZE]
+| output: count(*)
+| group by: rank() % 10
+| row-size=16B cardinality=10 (from HBO, key ignores partition constants)
+|
+02:ANALYTIC
+| functions: rank()
+| order by: id ASC
+| window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+| row-size=29B cardinality=620
+|
+04:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: id ASC
+|
+01:SORT
+| order by: id ASC
+| row-size=21B cardinality=620
+|
+00:SCAN HDFS [functional.alltypes]
+ partition predicates: `month` = 2
+ HDFS partitions=2/24 files=2 size=36.51KB
+ predicates: string_col != 'test_agg_over_analytic'
+ row-size=21B cardinality=620 (from HBO, key ignores partition constants)
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/hbo-analytic-join.test b/testdata/workloads/functional-query/queries/QueryTest/hbo-analytic-join.test
new file mode 100644
index 0000000000..675a51f59b
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/hbo-analytic-join.test
@@ -0,0 +1,229 @@
+====
+---- QUERY
+# EXPLAIN the analytic query that has been executed.
+EXPLAIN select straight_join t1.int_col, t1.rnk from
+ (select int_col, rank() over (order by id asc) as rnk
+ from functional.alltypes
+ where month=1 and string_col != 'test_join_over_analytic') t1,
+ (select int_col, rank() over (order by id desc) as rnk
+ from functional.alltypestiny
+ where month=1 and string_col != 'test_join_over_analytic') t2
+where t1.int_col = t2.int_col and t1.rnk = t2.rnk
+---- PLAN
+PLAN-ROOT SINK
+|
+06:HASH JOIN [INNER JOIN, BROADCAST]
+| hash predicates: int_col = int_col, rank() = rank()
+| row-size=66B cardinality=0 (from HBO)
+|
+|--09:EXCHANGE [UNPARTITIONED]
+| |
+| 05:ANALYTIC
+| | functions: rank()
+| | order by: id DESC
+| | window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+| | row-size=33B cardinality=2
+| |
+| 08:MERGING-EXCHANGE [UNPARTITIONED]
+| | order by: id DESC
+| |
+| 04:SORT
+| | order by: id DESC
+| | row-size=25B cardinality=2
+| |
+| 03:SCAN HDFS [functional.alltypestiny]
+| partition predicates: `month` = 1
+| HDFS partitions=1/4 files=1 size=115B
+| predicates: string_col != 'test_join_over_analytic'
+| row-size=25B cardinality=2 (from HBO)
+|
+02:ANALYTIC
+| functions: rank()
+| order by: id ASC
+| window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+| row-size=33B cardinality=620
+|
+07:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: id ASC
+|
+01:SORT
+| order by: id ASC
+| row-size=25B cardinality=620
+|
+00:SCAN HDFS [functional.alltypes]
+ partition predicates: `month` = 1
+ HDFS partitions=2/24 files=2 size=40.32KB
+ predicates: string_col != 'test_join_over_analytic'
+ row-size=25B cardinality=620 (from HBO)
+====
+---- QUERY
+# Change the JOIN order
+EXPLAIN select straight_join t1.int_col, t1.rnk from
+ (select int_col, rank() over (order by id desc) as rnk
+ from functional.alltypestiny
+ where month=1 and string_col != 'test_join_over_analytic') t2,
+ (select int_col, rank() over (order by id asc) as rnk
+ from functional.alltypes
+ where month=1 and string_col != 'test_join_over_analytic') t1
+where t1.int_col = t2.int_col and t1.rnk = t2.rnk
+---- PLAN
+PLAN-ROOT SINK
+|
+06:HASH JOIN [INNER JOIN, BROADCAST]
+| hash predicates: int_col = int_col, rank() = rank()
+| row-size=66B cardinality=0 (from HBO)
+|
+|--09:EXCHANGE [UNPARTITIONED]
+| |
+| 05:ANALYTIC
+| | functions: rank()
+| | order by: id ASC
+| | window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+| | row-size=33B cardinality=620
+| |
+| 08:MERGING-EXCHANGE [UNPARTITIONED]
+| | order by: id ASC
+| |
+| 04:SORT
+| | order by: id ASC
+| | row-size=25B cardinality=620
+| |
+| 03:SCAN HDFS [functional.alltypes]
+| partition predicates: `month` = 1
+| HDFS partitions=2/24 files=2 size=40.32KB
+| predicates: string_col != 'test_join_over_analytic'
+| row-size=25B cardinality=620 (from HBO)
+|
+02:ANALYTIC
+| functions: rank()
+| order by: id DESC
+| window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+| row-size=33B cardinality=2
+|
+07:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: id DESC
+|
+01:SORT
+| order by: id DESC
+| row-size=25B cardinality=2
+|
+00:SCAN HDFS [functional.alltypestiny]
+ partition predicates: `month` = 1
+ HDFS partitions=1/4 files=1 size=115B
+ predicates: string_col != 'test_join_over_analytic'
+ row-size=25B cardinality=2 (from HBO)
+====
+---- QUERY
+# Changing the analytic functions, the JOIN node have no HBO stats
+EXPLAIN select straight_join t1.int_col, t1.rnk from
+ (select int_col, rank() over (order by bigint_col, id asc) as rnk
+ from functional.alltypes
+ where month=1 and string_col != 'test_join_over_analytic') t1,
+ (select int_col, rank() over (order by bigint_col, id desc) as rnk
+ from functional.alltypestiny
+ where month=1 and string_col != 'test_join_over_analytic') t2
+where t1.int_col = t2.int_col and t1.rnk = t2.rnk
+---- PLAN
+PLAN-ROOT SINK
+|
+06:HASH JOIN [INNER JOIN, BROADCAST]
+| hash predicates: int_col = int_col, rank() = rank()
+| row-size=82B cardinality=1.24K
+|
+|--09:EXCHANGE [UNPARTITIONED]
+| |
+| 05:ANALYTIC
+| | functions: rank()
+| | order by: bigint_col ASC, id DESC
+| | window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+| | row-size=41B cardinality=2
+| |
+| 08:MERGING-EXCHANGE [UNPARTITIONED]
+| | order by: bigint_col ASC, id DESC
+| |
+| 04:SORT
+| | order by: bigint_col ASC, id DESC
+| | row-size=33B cardinality=2
+| |
+| 03:SCAN HDFS [functional.alltypestiny]
+| partition predicates: `month` = 1
+| HDFS partitions=1/4 files=1 size=115B
+| predicates: string_col != 'test_join_over_analytic'
+| row-size=33B cardinality=2 (from HBO)
+|
+02:ANALYTIC
+| functions: rank()
+| order by: bigint_col ASC, id ASC
+| window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+| row-size=41B cardinality=620
+|
+07:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: bigint_col ASC, id ASC
+|
+01:SORT
+| order by: bigint_col ASC, id ASC
+| row-size=33B cardinality=620
+|
+00:SCAN HDFS [functional.alltypes]
+ partition predicates: `month` = 1
+ HDFS partitions=2/24 files=2 size=40.32KB
+ predicates: string_col != 'test_join_over_analytic'
+ row-size=33B cardinality=620 (from HBO)
+====
+---- QUERY
+# Change the partition predicate to month=2
+EXPLAIN select straight_join t1.int_col, t1.rnk from
+ (select int_col, rank() over (order by id asc) as rnk
+ from functional.alltypes
+ where month=2 and string_col != 'test_join_over_analytic') t1,
+ (select int_col, rank() over (order by id desc) as rnk
+ from functional.alltypestiny
+ where month=2 and string_col != 'test_join_over_analytic') t2
+where t1.int_col = t2.int_col and t1.rnk = t2.rnk
+---- PLAN
+PLAN-ROOT SINK
+|
+06:HASH JOIN [INNER JOIN, BROADCAST]
+| hash predicates: int_col = int_col, rank() = rank()
+| row-size=66B cardinality=0 (from HBO, key ignores partition constants)
+|
+|--09:EXCHANGE [UNPARTITIONED]
+| |
+| 05:ANALYTIC
+| | functions: rank()
+| | order by: id DESC
+| | window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+| | row-size=33B cardinality=2
+| |
+| 08:MERGING-EXCHANGE [UNPARTITIONED]
+| | order by: id DESC
+| |
+| 04:SORT
+| | order by: id DESC
+| | row-size=25B cardinality=2
+| |
+| 03:SCAN HDFS [functional.alltypestiny]
+| partition predicates: `month` = 2
+| HDFS partitions=1/4 files=1 size=115B
+| predicates: string_col != 'test_join_over_analytic'
+| row-size=25B cardinality=2 (from HBO, key ignores partition constants)
+|
+02:ANALYTIC
+| functions: rank()
+| order by: id ASC
+| window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+| row-size=33B cardinality=620
+|
+07:MERGING-EXCHANGE [UNPARTITIONED]
+| order by: id ASC
+|
+01:SORT
+| order by: id ASC
+| row-size=25B cardinality=620
+|
+00:SCAN HDFS [functional.alltypes]
+ partition predicates: `month` = 2
+ HDFS partitions=2/24 files=2 size=36.51KB
+ predicates: string_col != 'test_join_over_analytic'
+ row-size=25B cardinality=620 (from HBO, key ignores partition constants)
+====
diff --git a/tests/query_test/test_hbo.py b/tests/query_test/test_hbo.py
index d132e69984..4aab9599b6 100644
--- a/tests/query_test/test_hbo.py
+++ b/tests/query_test/test_hbo.py
@@ -460,3 +460,31 @@ class TestHBO(HboTestMixin, ImpalaTestSuite):
where string_col != 'test_analytic_topn') t
where rnk <= 5""")
self._run_hbo_explains('QueryTest/hbo-analytic-topn')
+
+ def test_agg_over_analytic_cardinality(self):
+ """Test an Aggregation over analytic nodes using the analytic results as grouping
+ keys. AnalyticEvalNode won't use HBO stats. But its parent, the AggregationNode,
+ should be able to use HBO stats."""
+ self.client.set_configuration(QUERY_OPTIONS)
+ self.execute_query("""
+ select count(*), rnk % 10 from
+ (select int_col, rank() over (order by id asc) as rnk
+ from functional.alltypes
+ where month=1 and string_col != 'test_agg_over_analytic') t
+ group by rnk % 10""")
+ self._run_hbo_explains('QueryTest/hbo-analytic-agg')
+
+ def test_join_over_analytic_cardinality(self):
+ """Test a JOIN over analytic nodes joining the analytic results. AnalyticEvalNodes
+ won't use HBO stats. But its parent, the JoinNode should be able to use HBO stats."""
+ self.client.set_configuration(QUERY_OPTIONS)
+ self.execute_query("""
+ select straight_join t1.int_col, t1.rnk from
+ (select int_col, rank() over (order by id asc) as rnk
+ from functional.alltypes
+ where month=1 and string_col != 'test_join_over_analytic') t1,
+ (select int_col, rank() over (order by id desc) as rnk
+ from functional.alltypestiny
+ where month=1 and string_col != 'test_join_over_analytic') t2
+ where t1.int_col = t2.int_col and t1.rnk = t2.rnk""")
+ self._run_hbo_explains('QueryTest/hbo-analytic-join')