You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2026/10/02 19:55:52 UTC
(impala) branch master updated: IMPALA-15394: Add missing scan properties to the HBO scan key
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new c9c8c9df8 IMPALA-15394: Add missing scan properties to the HBO scan key
c9c8c9df8 is described below
commit c9c8c9df8edabc8415e86e2214652a73a50ae94b
Author: Aleksandr Efimov <dz...@gmail.com>
AuthorDate: Tue Sep 22 14:42:53 2026 +0300
IMPALA-15394: Add missing scan properties to the HBO scan key
HdfsScanNode.generateHboKeyString() keys a scan by its table and
conjuncts. Some scans of the same table and conjuncts return different
rows, and their input stats match too, so HBO gives one the cardinality
of the other:
- An optimized count(*) scan returns one row per file or row group.
After "SELECT count(*) FROM t" on a Parquet table with 15 files and
13.66M rows, "SELECT * FROM t" is planned with cardinality=15 (from
HBO), and every estimate above the scan follows.
- A partition key scan, e.g. "SELECT DISTINCT year FROM t", returns one
row per scan range.
- A sampled scan (TABLESAMPLE) reads only part of the files, while its
input rows count every partition with a sampled file (or all files of
an Iceberg table).
- Conjuncts on the items of a collection, together with the
IsNotEmptyPredicate of the join, filter the rows of the scan. On
functional_parquet.complextypestbl the scan returned 2 rows for
"a.item > 1" and 0 for "a.item > 100" under the same key.
This patch adds <COUNT_STAR>, <PARTITION_KEY_SCAN> and <SAMPLE:percent>
markers and the canonicalized collection conjuncts to the scan key. The
collection is named by its canonical path, so the key doesn't depend on
the aliases. The seed of a sample is left out: it picks other files but
reads the same share of the table.
Testing:
- Added HboKeyStringTest.testCountStarScanNodeKey,
testPartitionKeyAndSampledScanNodeKeys and
testCollectionConjunctsInScanNodeKey. Removing each marker or the
collection conjuncts, or naming the collection by its alias, fails
the test that covers it.
- Added e2e cases to test_hbo.py: a regular scan after a count(*) scan
or a partition key scan, and item conjuncts of a joined collection.
Without this patch each gets the cardinality of the other query from
HBO.
Change-Id: I03183aa998f90bf834dba4ce9ad2f2e57fef1140
Assisted-by: claude-opus-5 (Claude Code)
Reviewed-on: http://gerrit.cloudera.org:8080/24913
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../org/apache/impala/planner/HdfsScanNode.java | 20 +++++++
.../apache/impala/planner/HboKeyStringTest.java | 63 ++++++++++++++++++++
.../queries/QueryTest/hbo-collection-scan.test | 69 ++++++++++++++++++++++
.../queries/QueryTest/hbo-count-star-scan.test | 26 ++++++++
.../queries/QueryTest/hbo-partition-key-scan.test | 37 ++++++++++++
tests/query_test/test_hbo.py | 19 ++++++
6 files changed, 234 insertions(+)
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
index ba496475c..091e28090 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
@@ -1954,6 +1954,15 @@ public class HdfsScanNode extends ScanNode {
} else {
sb.append(tbl_.getFullName());
}
+ // An optimized count(*) scan returns one row per file or row group, and a partition
+ // key scan returns one row per scan range, not the table's rows, while their input
+ // stats are those of a regular scan of the same table. A sampled scan reads only
+ // part of the files. So none of them may share a key with a regular scan.
+ if (countStarSlot_ != null) sb.append("|<COUNT_STAR>");
+ if (isPartitionKeyScan_) sb.append("|<PARTITION_KEY_SCAN>");
+ if (sampleParams_ != null) {
+ sb.append("|<SAMPLE:").append(sampleParams_.getPercentBytes()).append(">");
+ }
// Canonicalize partition conjuncts and regular conjuncts.
List<String> partConjStrings =
@@ -1969,6 +1978,17 @@ public class HdfsScanNode extends ScanNode {
sb.append("|").append(s);
LOG.trace("HBO CONJUNCT STR ({}, {}): {}", statsType, strategy, s);
}
+ // Conjuncts on collection items filter the items, and with an IsNotEmptyPredicate
+ // also the rows of this scan.
+ List<String> collConjStrings = new ArrayList<>();
+ for (Map.Entry<TupleDescriptor, List<Expr>> entry : collectionConjuncts_.entrySet()) {
+ String path = String.join(".", entry.getKey().getPath().getCanonicalPath());
+ List<String> itemConjStrings = ExprCanonicalizer.canonicalizeScanConjuncts(
+ entry.getValue(), tbl_, strategy);
+ collConjStrings.add(path + ":[" + String.join(",", itemConjStrings) + "]");
+ }
+ Collections.sort(collConjStrings);
+ for (String s: collConjStrings) sb.append("|").append(s);
return sb.toString();
}
diff --git a/fe/src/test/java/org/apache/impala/planner/HboKeyStringTest.java b/fe/src/test/java/org/apache/impala/planner/HboKeyStringTest.java
index f9cc78c87..2a109ffa5 100644
--- a/fe/src/test/java/org/apache/impala/planner/HboKeyStringTest.java
+++ b/fe/src/test/java/org/apache/impala/planner/HboKeyStringTest.java
@@ -136,6 +136,69 @@ public class HboKeyStringTest extends FrontendTestBase {
assertEquals(expectedIgnorePartConstKey, ignorePartConstkey);
}
+ /**
+ * Returns the key of the scan in a query with a single scan, which is plan node 0.
+ */
+ private String singleScanKey(String query) throws ImpalaException {
+ return collectPlanNodesInDistributedPlan(query).get(0).generateHboKeyString(
+ THboStatsType.CARDINALITY, CanonicalizationStrategy.EXPR_REWRITE);
+ }
+
+ /**
+ * An optimized count(*) scan outputs one row per file or row group, so it must not
+ * share a key with a regular scan of the same table and conjuncts.
+ */
+ @Test
+ public void testCountStarScanNodeKey() throws ImpalaException {
+ assertEquals("CARDINALITY:ScanNode:functional_parquet.alltypes|<COUNT_STAR>|" +
+ "`year` = 2009", singleScanKey(
+ "SELECT count(*) FROM functional_parquet.alltypes WHERE year = 2009"));
+ assertEquals("CARDINALITY:ScanNode:functional_parquet.alltypes|`year` = 2009",
+ singleScanKey(
+ "SELECT max(id) FROM functional_parquet.alltypes WHERE year = 2009"));
+ }
+
+ /**
+ * A partition key scan outputs one row per scan range and a sampled scan reads only
+ * part of the files, so neither may share a key with a regular scan of the same table
+ * and conjuncts.
+ */
+ @Test
+ public void testPartitionKeyAndSampledScanNodeKeys() throws ImpalaException {
+ String alltypesKey = "CARDINALITY:ScanNode:functional_parquet.alltypes";
+ assertEquals(alltypesKey,
+ singleScanKey("SELECT year FROM functional_parquet.alltypes"));
+ assertEquals(alltypesKey + "|<PARTITION_KEY_SCAN>",
+ singleScanKey("SELECT DISTINCT year FROM functional_parquet.alltypes"));
+ assertEquals(alltypesKey + "|<SAMPLE:10>", singleScanKey("SELECT year FROM " +
+ "functional_parquet.alltypes TABLESAMPLE SYSTEM(10) REPEATABLE(1)"));
+
+ String icebergKey = "CARDINALITY:ScanNode:functional_parquet.iceberg_partitioned";
+ assertEquals(icebergKey,
+ singleScanKey("SELECT id FROM functional_parquet.iceberg_partitioned"));
+ assertEquals(icebergKey + "|<SAMPLE:10>", singleScanKey("SELECT id FROM " +
+ "functional_parquet.iceberg_partitioned TABLESAMPLE SYSTEM(10) REPEATABLE(1)"));
+ }
+
+ /**
+ * With an IsNotEmptyPredicate, conjuncts on the items of a collection also filter the
+ * rows of the scan, so they are part of its key. The collection is named by its path,
+ * not by its alias.
+ */
+ @Test
+ public void testCollectionConjunctsInScanNodeKey() throws ImpalaException {
+ String tblKey =
+ "CARDINALITY:ScanNode:functional_parquet.complextypestbl|!empty(int_array)";
+ assertEquals(tblKey, singleScanKey("SELECT c.id FROM " +
+ "functional_parquet.complextypestbl c, c.int_array a"));
+ assertEquals(tblKey + "|functional_parquet.complextypestbl.int_array:[item > 1]",
+ singleScanKey("SELECT c.id FROM functional_parquet.complextypestbl c, " +
+ "c.int_array a WHERE a.item > 1"));
+ assertEquals(tblKey + "|functional_parquet.complextypestbl.int_array:[item > 100]",
+ singleScanKey("SELECT t.id FROM functional_parquet.complextypestbl t, " +
+ "t.int_array x WHERE x.item > 100"));
+ }
+
@Test
public void testAggregationNodeKeys() throws ImpalaException {
String query = "SELECT month, count(id) FROM functional.alltypes " +
diff --git a/testdata/workloads/functional-query/queries/QueryTest/hbo-collection-scan.test b/testdata/workloads/functional-query/queries/QueryTest/hbo-collection-scan.test
index 147964ddb..b469e89b7 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/hbo-collection-scan.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/hbo-collection-scan.test
@@ -513,3 +513,72 @@ PLAN-ROOT SINK
predicates: item > 0, pos > 1
row-size=12B cardinality=3 (from HBO)
====
+---- QUERY
+EXPLAIN SELECT count(*) FROM functional_parquet.complextypestbl c, c.int_array a
+WHERE a.item > 1
+---- PLAN
+PLAN-ROOT SINK
+|
+07:AGGREGATE [FINALIZE]
+| output: count:merge(*)
+| row-size=8B cardinality=1
+|
+06:EXCHANGE [UNPARTITIONED]
+|
+05:AGGREGATE
+| output: count(*)
+| row-size=8B cardinality=2
+|
+01:SUBPLAN
+| row-size=16B cardinality=20
+|
+|--04:NESTED LOOP JOIN [CROSS JOIN]
+| | row-size=16B cardinality=10
+| |
+| |--02:SINGULAR ROW SRC
+| | row-size=12B cardinality=1
+| |
+| 03:UNNEST [c.int_array a]
+| row-size=0B cardinality=10
+|
+00:SCAN HDFS [functional_parquet.complextypestbl c]
+ HDFS partitions=1/1 files=2 size=6.92KB
+ predicates: !empty(c.int_array)
+ predicates on a: a.item > 1
+ row-size=12B cardinality=2 (from HBO)
+====
+---- QUERY
+# Other conjuncts on the collection items, so no stats from HBO (IMPALA-15394).
+EXPLAIN SELECT count(*) FROM functional_parquet.complextypestbl c, c.int_array a
+WHERE a.item > 100
+---- PLAN
+PLAN-ROOT SINK
+|
+07:AGGREGATE [FINALIZE]
+| output: count:merge(*)
+| row-size=8B cardinality=1
+|
+06:EXCHANGE [UNPARTITIONED]
+|
+05:AGGREGATE
+| output: count(*)
+| row-size=8B cardinality=2
+|
+01:SUBPLAN
+| row-size=16B cardinality=44.00K
+|
+|--04:NESTED LOOP JOIN [CROSS JOIN]
+| | row-size=16B cardinality=10
+| |
+| |--02:SINGULAR ROW SRC
+| | row-size=12B cardinality=1
+| |
+| 03:UNNEST [c.int_array a]
+| row-size=0B cardinality=10
+|
+00:SCAN HDFS [functional_parquet.complextypestbl c]
+ HDFS partitions=1/1 files=2 size=6.92KB
+ predicates: !empty(c.int_array)
+ predicates on a: a.item > 100
+ row-size=12B cardinality=4.40K
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/hbo-count-star-scan.test b/testdata/workloads/functional-query/queries/QueryTest/hbo-count-star-scan.test
new file mode 100644
index 000000000..883ee0790
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/hbo-count-star-scan.test
@@ -0,0 +1,26 @@
+====
+---- QUERY
+EXPLAIN SELECT count(*) FROM tpch_parquet.orders
+---- PLAN
+PLAN-ROOT SINK
+|
+01:AGGREGATE [FINALIZE]
+| output: sum_init_zero(tpch_parquet.orders.stats: num_rows)
+| row-size=8B cardinality=1 (from HBO)
+|
+00:SCAN HDFS [tpch_parquet.orders]
+ HDFS partitions=1/1 files=2 size=54.21MB
+ row-size=8B cardinality=2 (from HBO)
+====
+---- QUERY
+# The regular scan must not get the cardinality of the count(*) scan (IMPALA-15394).
+EXPLAIN SELECT o_orderkey FROM tpch_parquet.orders
+---- PLAN
+PLAN-ROOT SINK
+|
+01:EXCHANGE [UNPARTITIONED]
+|
+00:SCAN HDFS [tpch_parquet.orders]
+ HDFS partitions=1/1 files=2 size=54.21MB
+ row-size=8B cardinality=1.50M
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/hbo-partition-key-scan.test b/testdata/workloads/functional-query/queries/QueryTest/hbo-partition-key-scan.test
new file mode 100644
index 000000000..b172209c0
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/hbo-partition-key-scan.test
@@ -0,0 +1,37 @@
+====
+---- QUERY
+EXPLAIN SELECT DISTINCT year, month FROM functional.alltypes WHERE month > 10
+---- PLAN
+PLAN-ROOT SINK
+|
+04:EXCHANGE [UNPARTITIONED]
+|
+03:AGGREGATE [FINALIZE]
+| group by: `year`, `month`
+| row-size=8B cardinality=4 (from HBO)
+|
+02:EXCHANGE [HASH(`year`,`month`)]
+|
+01:AGGREGATE [STREAMING]
+| group by: `year`, `month`
+| row-size=8B cardinality=4 (from HBO)
+|
+00:SCAN HDFS [functional.alltypes]
+ partition predicates: `month` > 10
+ HDFS partitions=4/24 files=4 size=80.14KB
+ partition key scan
+ row-size=8B cardinality=4 (from HBO)
+====
+---- QUERY
+# The regular scan must not get the cardinality of the partition key scan (IMPALA-15394).
+EXPLAIN SELECT id FROM functional.alltypes WHERE month > 10
+---- PLAN
+PLAN-ROOT SINK
+|
+01:EXCHANGE [UNPARTITIONED]
+|
+00:SCAN HDFS [functional.alltypes]
+ partition predicates: `month` > 10
+ HDFS partitions=4/24 files=4 size=80.14KB
+ row-size=4B cardinality=1.22K
+====
diff --git a/tests/query_test/test_hbo.py b/tests/query_test/test_hbo.py
index 1b6d4761f..905de322b 100644
--- a/tests/query_test/test_hbo.py
+++ b/tests/query_test/test_hbo.py
@@ -234,8 +234,27 @@ class TestHBO(ImpalaTestSuite):
where = " WHERE " + where_expr if len(where_expr) > 0 else ""
# Run query to populate HBO stats
self.execute_query(stmt % "count(*)" + where)
+ # Conjuncts on the items of a joined collection also filter the rows of the table
+ # scan, so "a.item > 100" must not get the stats of "a.item > 1".
+ self.execute_query("SELECT count(*) FROM functional_parquet.complextypestbl c, "
+ "c.int_array a WHERE a.item > 1")
self._run_hbo_explains('QueryTest/hbo-collection-scan')
+ def test_count_star_scan_cardinality(self):
+ self.client.set_configuration(QUERY_OPTIONS)
+ # An optimized count(*) scan returns one row per file, so a regular scan of the same
+ # table and conjuncts must not get its cardinality. Use a parquet table with stats.
+ self.execute_query("SELECT count(*) FROM tpch_parquet.orders")
+ self._run_hbo_explains('QueryTest/hbo-count-star-scan')
+
+ def test_partition_key_scan_cardinality(self):
+ self.client.set_configuration(QUERY_OPTIONS)
+ # A partition key scan returns one row per scan range, so a regular scan of the same
+ # table and conjuncts must not get its cardinality.
+ self.execute_query(
+ "SELECT DISTINCT year, month FROM functional.alltypes WHERE month > 10")
+ self._run_hbo_explains('QueryTest/hbo-partition-key-scan')
+
def test_iceberg_scan_cardinality(self):
self.client.set_configuration(QUERY_OPTIONS)
self.execute_query(