You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by cs...@apache.org on 2026/07/17 08:41:29 UTC
(impala) branch master updated: IMPALA-15135: Calcite planner: added support for Kudu tables
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 3dcd7c493 IMPALA-15135: Calcite planner: added support for Kudu tables
3dcd7c493 is described below
commit 3dcd7c4936abc22d10368c6dca6b0bd973010027
Author: Steve Carlin <sc...@cloudera.com>
AuthorDate: Tue Jun 23 13:41:13 2026 -0700
IMPALA-15135: Calcite planner: added support for Kudu tables
This commit enables the Calcite planner to support queries on
Kudu tables. Highlights include:
- The Kudu tables can have an auto_incrementing_id column that
does not show up when a "select *" is done on the query. Special
logic has been added to support this. If the user expliclity selects
the auto_incrementing_id column, the query will fall back to the
original planner (IMPALA-15136 has been filed for this).
- A small refactor was done for PrunedPartitionHelper to handle
Kudu tables because it does not support code in the HdfsPartitionPruner.
- A small refactor was done in CalciteDb to handle FeKuduTable
- IMPALA-15134 has been filed on an issue where the cardinality on
the KuduScan table is different from the original planner.
- Added some code in the test framework to handle the section
CALCITE_PLANNER_RUNTIME_PROFILE for when a table format is specified.
Change-Id: Ie22a66f0edad195cade74a8447efc83a8ce526f7
Reviewed-on: http://gerrit.cloudera.org:8080/24528
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Steve Carlin <sc...@cloudera.com>
---
.../org/apache/impala/analysis/SelectStmt.java | 6 +---
.../org/apache/impala/catalog/FeKuduTable.java | 14 ++++++++
.../java/org/apache/impala/catalog/FeTable.java | 11 +++++++
.../impala/calcite/rel/node/ImpalaHdfsScanRel.java | 15 ++++++---
.../calcite/rel/util/PrunedPartitionHelper.java | 20 ++++++++++++
.../apache/impala/calcite/schema/CalciteDb.java | 38 +++++++++++++---------
.../impala/calcite/schema/CalciteIcebergTable.java | 2 +-
.../apache/impala/calcite/schema/CalciteTable.java | 31 +++++++++---------
.../calcite/service/ImpalaSqlValidatorImpl.java | 4 +--
.../impala/calcite/service/UnsupportedChecker.java | 6 ++++
...u_runtime_filter_with_timestamp_conversion.test | 6 ++--
.../QueryTest/partitioned-top-n-high-ndv.test | 6 ++--
.../queries/QueryTest/runtime_filters.test | 6 ++++
tests/common/impala_test_suite.py | 8 ++++-
tests/util/test_file_parser.py | 6 ++--
15 files changed, 126 insertions(+), 53 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
index 687bb93fe..1d2ed487c 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
@@ -44,7 +44,6 @@ import org.apache.impala.catalog.FeIcebergTable.Utils;
import org.apache.impala.catalog.FeKuduTable;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.FeView;
-import org.apache.impala.catalog.KuduColumn;
import org.apache.impala.catalog.MapType;
import org.apache.impala.catalog.StructField;
import org.apache.impala.catalog.StructType;
@@ -902,10 +901,7 @@ public class SelectStmt extends QueryStmt {
// table. Expand the '*' based on the Hive-column order.
TupleDescriptor tupleDesc = resolvedPath.destTupleDesc();
FeTable table = tupleDesc.getTable();
- for (Column c: table.getColumnsInHiveOrder()) {
- if (c.isHidden()) continue;
- // Omit auto-incrementing column for Kudu table since it's a hidden column.
- if (c instanceof KuduColumn && ((KuduColumn)c).isAutoIncrementing()) continue;
+ for (Column c: table.getStarColumns()) {
addStarExpandedPath(selectListItem, resolvedPath, c.getName());
}
} else {
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeKuduTable.java b/fe/src/main/java/org/apache/impala/catalog/FeKuduTable.java
index c7bee21a3..5423a11e8 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeKuduTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeKuduTable.java
@@ -78,6 +78,20 @@ public interface FeKuduTable extends FeTable {
*/
List<KuduPartitionParam> getPartitionBy();
+ /**
+ * @return an Iterable for a list of the columns used when a select * is called. This
+ * will also be in "hive" order by default. For Kudu, auto_incrementing columns
+ * and hidden columns are not output with select *.
+ */
+ @Override
+ default Iterable<Column> getStarColumns() {
+ return getColumnsInHiveOrder().stream()
+ .map(c -> (KuduColumn)c)
+ .filter(c -> !c.isHidden() && !c.isAutoIncrementing())
+ .map(c -> (Column)c)
+ ::iterator;
+ }
+
/**
* Utility functions for acting on FeKuduTable.
*
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeTable.java b/fe/src/main/java/org/apache/impala/catalog/FeTable.java
index e458482d3..c560ca107 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeTable.java
@@ -126,6 +126,17 @@ public interface FeTable {
*/
List<Column> getColumnsInHiveOrder();
+ /**
+ * @return an Iterable for a list of the columns used when a select * is called. This
+ * will also be in "hive" order by default. Hidden columns are not output with
+ * select *.
+ */
+ default Iterable<Column> getStarColumns() {
+ return getColumnsInHiveOrder().stream()
+ .filter(c -> !c.isHidden())
+ ::iterator;
+ }
+
/**
* @return a list of the column names ordered by position.
*/
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaHdfsScanRel.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaHdfsScanRel.java
index 93506a0b7..4222448ce 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaHdfsScanRel.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaHdfsScanRel.java
@@ -36,13 +36,14 @@ import org.apache.impala.calcite.schema.CalciteTable;
import org.apache.impala.calcite.util.SimplifiedAnalyzer;
import org.apache.impala.catalog.Column;
import org.apache.impala.catalog.FeFsPartition;
-import org.apache.impala.catalog.FeFsTable;
import org.apache.impala.catalog.FeIcebergTable;
+import org.apache.impala.catalog.FeKuduTable;
import org.apache.impala.common.IcebergPredicateConverter;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.UnsupportedFeatureException;
import org.apache.impala.planner.HdfsScanNode;
import org.apache.impala.planner.IcebergScanPlanner;
+import org.apache.impala.planner.KuduScanNode;
import org.apache.impala.planner.PlanNode;
import org.apache.impala.planner.PlanNodeId;
import org.apache.impala.planner.ScanNode;
@@ -87,7 +88,7 @@ public class ImpalaHdfsScanRel extends TableScan
context.filterCondition_, outputExprs, getCluster().getRexBuilder(),
analyzer);
- PrunedPartitionHelper pph = new PrunedPartitionHelper(table, converter,
+ PrunedPartitionHelper pph = PrunedPartitionHelper.create(table, converter,
tupleDesc, getCluster().getRexBuilder(), context.ctx_.getRootAnalyzer());
List<? extends FeFsPartition> impalaPartitions = pph.getPrunedPartitions();
@@ -124,6 +125,10 @@ public class ImpalaHdfsScanRel extends TableScan
physicalNode =
SingleNodePlanner.createOptimizedPartitionUnionNode(nodeId, impalaPartitions,
tupleDesc, analyzer);
+ } else if (table.getFeTable() instanceof FeKuduTable) {
+ List<Expr> allConjuncts = new ArrayList<>(filterConjuncts);
+ physicalNode =
+ new KuduScanNode(nodeId, tupleDesc, allConjuncts, null, baseTblRef);
} else if (table instanceof CalciteIcebergTable) {
CalciteIcebergTable iceTable = (CalciteIcebergTable) table;
// Can't use filterConjuncts because it is in an immutable list and the iceberg
@@ -172,7 +177,6 @@ public class ImpalaHdfsScanRel extends TableScan
private List<Expr> createScanOutputExprs(List<SlotDescriptor> slotDescs)
throws ImpalaException {
CalciteTable calciteTable = (CalciteTable) getTable();
- FeFsTable table = calciteTable.getFeFsTable();
// IMPALA-12961: The output expressions are contained in a list which
// may have holes in it (if the table scan column is not in the output).
// The width of the list must include all columns, including the acid ones,
@@ -284,8 +288,9 @@ public class ImpalaHdfsScanRel extends TableScan
return false;
} else {
for (Expr conjunct : filterConjuncts) {
- if (!canConvertIcebergPredicate((FeIcebergTable) table.getFeFsTable(),
- conjunct, analyzer)) {
+ CalciteIcebergTable icebergTable = (CalciteIcebergTable) table;
+ if (!canConvertIcebergPredicate(icebergTable.getFeIcebergTable(), conjunct,
+ analyzer)) {
return false;
}
}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/PrunedPartitionHelper.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/PrunedPartitionHelper.java
index 44e828e6f..9e5219b00 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/PrunedPartitionHelper.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/PrunedPartitionHelper.java
@@ -25,6 +25,7 @@ import org.apache.impala.analysis.TupleDescriptor;
import org.apache.impala.calcite.schema.CalciteTable;
import org.apache.impala.calcite.schema.CalciteIcebergTable;
import org.apache.impala.catalog.FeFsPartition;
+import org.apache.impala.catalog.FeKuduTable;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.Pair;
import org.apache.impala.planner.HdfsPartitionPruner;
@@ -45,6 +46,12 @@ public class PrunedPartitionHelper {
private final List<Expr> nonPartitionedConjuncts_;
+ public PrunedPartitionHelper(ExprConjunctsConverter converter) {
+ prunedPartitions_ = new ArrayList<>();
+ partitionedConjuncts_ = new ArrayList<>();
+ nonPartitionedConjuncts_ = converter.getImpalaConjuncts();
+ }
+
public PrunedPartitionHelper(CalciteTable table,
ExprConjunctsConverter converter, TupleDescriptor tupleDesc,
RexBuilder rexBuilder,
@@ -94,4 +101,17 @@ public class PrunedPartitionHelper {
public List<Expr> getNonPartitionedConjuncts() {
return nonPartitionedConjuncts_;
}
+
+ public static PrunedPartitionHelper create(CalciteTable table,
+ ExprConjunctsConverter converter, TupleDescriptor tupleDesc,
+ RexBuilder rexBuilder,
+ Analyzer analyzer) throws ImpalaException {
+ if ((table.getFeTable() instanceof FeKuduTable)) {
+ // Kudu tables don't use the PrunedPartition class. The conjuncts don't
+ // need to be split up into partitioned and non-partitioned conjuncts, so
+ // a simpler constructor is used.
+ return new PrunedPartitionHelper(converter);
+ }
+ return new PrunedPartitionHelper(table, converter, tupleDesc, rexBuilder, analyzer);
+ }
}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/CalciteDb.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/CalciteDb.java
index 84f391c0e..96ded22ae 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/CalciteDb.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/CalciteDb.java
@@ -31,6 +31,7 @@ import org.apache.impala.catalog.FeIcebergTable;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.FeView;
import org.apache.impala.catalog.HdfsTable;
+import org.apache.impala.catalog.FeKuduTable;
import org.apache.impala.catalog.local.LocalFsTable;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.UnsupportedFeatureException;
@@ -60,31 +61,36 @@ public class CalciteDb extends AbstractSchema {
this.reader_ = reader;
}
- public Builder addTable(String tableName, FeTable table,
+ public Builder addTable(String tableName, FeTable feTable,
Analyzer analyzer) throws ImpalaException {
if (tableMap_.containsKey(tableName)) return this;
- if (table instanceof LocalFsTable || table instanceof HdfsTable) {
- tableMap_.put(
- tableName.toLowerCase(), new CalciteTable(table, reader_, analyzer));
- return this;
+ Table table = createCalciteTable(feTable, analyzer);
+
+ if (table == null) {
+ throw new UnsupportedFeatureException("Table " + feTable.getFullName()
+ + " has unsupported type " + table.getClass().getSimpleName() + ".");
+ }
+ tableMap_.put(tableName.toLowerCase(), table);
+ return this;
+ }
+
+ private Table createCalciteTable(FeTable feTable, Analyzer analyzer)
+ throws ImpalaException {
+ if (feTable instanceof FeView) {
+ return createViewTable(feTable);
}
- if (table instanceof FeIcebergTable) {
- tableMap_.put(tableName.toLowerCase(),
- new CalciteIcebergTable((FeIcebergTable) table, reader_, analyzer));
- return this;
+ if (feTable instanceof LocalFsTable || feTable instanceof HdfsTable ||
+ feTable instanceof FeKuduTable) {
+ return new CalciteTable(feTable, reader_, analyzer);
}
- if (table instanceof FeView) {
- tableMap_.put(tableName.toLowerCase(), createViewTable(table));
- return this;
+ if (feTable instanceof FeIcebergTable) {
+ return new CalciteIcebergTable((FeIcebergTable) feTable, reader_, analyzer);
}
- throw new UnsupportedFeatureException("Table " + table.getFullName()
- + " has unsupported type " + table.getClass().getSimpleName()
- + ". The Calcite planner only supports "
- + "HdfsTable's and FeView's.");
+ return null;
}
private static ViewTable createViewTable(FeTable feTable) throws ImpalaException {
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/CalciteIcebergTable.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/CalciteIcebergTable.java
index 1a579ce51..6b1c3cbb2 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/CalciteIcebergTable.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/CalciteIcebergTable.java
@@ -50,7 +50,7 @@ public class CalciteIcebergTable extends CalciteTable {
}
public FeIcebergTable getFeIcebergTable() {
- return (FeIcebergTable) getFeFsTable();
+ return (FeIcebergTable) getFeTable();
}
@Override
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/CalciteTable.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/CalciteTable.java
index 9dce53872..801e739d9 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/CalciteTable.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/schema/CalciteTable.java
@@ -18,6 +18,7 @@
package org.apache.impala.calcite.schema;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import org.apache.calcite.config.CalciteConnectionConfig;
import org.apache.calcite.plan.RelOptAbstractTable;
@@ -47,8 +48,8 @@ import org.apache.impala.calcite.util.SimplifiedAnalyzer;
import org.apache.impala.catalog.Column;
import org.apache.impala.catalog.FeFsPartition;
import org.apache.impala.catalog.FeFsTable;
+import org.apache.impala.catalog.FeKuduTable;
import org.apache.impala.catalog.FeTable;
-import org.apache.impala.catalog.FeView;
import org.apache.impala.catalog.HdfsFileFormat;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.Pair;
@@ -66,7 +67,7 @@ import java.util.Set;
public class CalciteTable extends RelOptAbstractTable
implements Table, Prepare.PreparingTable {
- private final FeFsTable table_;
+ private final FeTable table_;
private final Map<Integer, Integer> impalaPositionMap_;
@@ -81,17 +82,18 @@ public class CalciteTable extends RelOptAbstractTable
public CalciteTable(FeTable table, CalciteCatalogReader reader,
Analyzer analyzer) throws ImpalaException {
super(reader, table.getName(), buildColumnsForRelDataType(table));
- this.table_ = (FeFsTable) table;
+ this.table_ = (FeTable) table;
this.qualifiedTableName_ = table.getTableName().toPath();
- this.columns_ = table.getColumnsInHiveOrder();
+ this.columns_ = Lists.newArrayList(table.getStarColumns());
this.impalaPositionMap_ = buildPositionMap();
this.analyzer_ = (SimplifiedAnalyzer) analyzer;
// TODO: If table_.getNumRows() is unknown (-1), this logic will load all partitions
// to compute estimation using HdfsEstimatedMissingTableStats. This is potentially
// expensive and should be avoided in local catalog mode.
- estimatedMissingStats_ = table_.getNumRows() < 0 ?
+ estimatedMissingStats_ = (table_ instanceof FeFsTable) && table_.getNumRows() < 0 ?
new HdfsEstimatedMissingTableStats(
- analyzer.getQueryOptions(), table_, table_.loadAllPartitions(), -1) :
+ analyzer.getQueryOptions(), (FeFsTable) table_,
+ ((FeFsTable) table_).loadAllPartitions(), -1) :
null;
checkIfTableIsSupported(table);
@@ -104,7 +106,7 @@ public class CalciteTable extends RelOptAbstractTable
RelDataTypeFactory.Builder builder = new RelDataTypeFactory.Builder(typeFactory);
// skip clustering columns, save them for the end
- for (Column column : table.getColumnsInHiveOrder()) {
+ for (Column column : table.getStarColumns()) {
if (column.getType().isComplexType()) {
throw new UnsupportedFeatureException(
"Calcite does not support complex types yet.");
@@ -117,11 +119,7 @@ public class CalciteTable extends RelOptAbstractTable
}
private void checkIfTableIsSupported(FeTable table) throws ImpalaException {
- if (table instanceof FeView) {
- throw new UnsupportedFeatureException("Views are not supported yet.");
- }
-
- if (!(table instanceof FeFsTable)) {
+ if (!(table instanceof FeFsTable) && !(table instanceof FeKuduTable)) {
String tableType = table.getClass().getSimpleName().replace("Table", "");
throw new UnsupportedFeatureException(tableType + " tables are not supported yet.");
}
@@ -153,7 +151,7 @@ public class CalciteTable extends RelOptAbstractTable
return impalaPair.first;
}
- public FeFsTable getFeFsTable() { return table_; }
+ public FeTable getFeTable() { return table_; }
@Override
public List<String> getQualifiedName() {
@@ -250,7 +248,7 @@ public class CalciteTable extends RelOptAbstractTable
Map<Integer, Integer> impalaPositionMap = new HashMap<>();
// skip clustering columns, save them for the end
int i = 0;
- for (Column column : table_.getColumnsInHiveOrder()) {
+ for (Column column : table_.getStarColumns()) {
impalaPositionMap.put(column.getPosition(), i);
i++;
}
@@ -271,7 +269,10 @@ public class CalciteTable extends RelOptAbstractTable
* needed to apply the count star optimization.
*/
public boolean canApplyCountStarOptimization() {
- Set<HdfsFileFormat> fileFormats = table_.getFileFormats();
+ if (!(table_ instanceof FeFsTable)) {
+ return false;
+ }
+ Set<HdfsFileFormat> fileFormats = ((FeFsTable)table_).getFileFormats();
if (fileFormats.size() != 1) {
return false;
}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/ImpalaSqlValidatorImpl.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/ImpalaSqlValidatorImpl.java
index e0d7cf251..82821b738 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/ImpalaSqlValidatorImpl.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/ImpalaSqlValidatorImpl.java
@@ -48,8 +48,8 @@ import org.apache.impala.calcite.schema.CalciteTable;
import org.apache.impala.calcite.schema.ImpalaViewTable;
import org.apache.impala.calcite.type.ImpalaTypeConverter;
import org.apache.impala.catalog.BuiltinsDb;
+import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.FeView;
-import org.apache.impala.catalog.FeFsTable;
import org.apache.impala.catalog.Type;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -135,7 +135,7 @@ public class ImpalaSqlValidatorImpl extends SqlValidatorImpl {
SqlValidatorTable validatorTable = fromNs.getTable();
if (validatorTable instanceof CalciteTable) {
- FeFsTable feFsTable = ((CalciteTable) validatorTable).getFeFsTable();
+ FeTable feFsTable = ((CalciteTable) validatorTable).getFeTable();
this.analyzer_.registerPrivReq(
builder -> builder.allOf(Privilege.SELECT)
.onColumn(feFsTable.getDb().getName(),
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/UnsupportedChecker.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/UnsupportedChecker.java
index 5d6f01abd..59b013bdd 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/UnsupportedChecker.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/UnsupportedChecker.java
@@ -48,6 +48,9 @@ public class UnsupportedChecker {
private static Pattern FILE_POSITION = Pattern.compile(".*\\bfile__position\\b.*",
Pattern.CASE_INSENSITIVE);
+ private static Pattern AUTO_INCREMENTING_ID =
+ Pattern.compile(".*\\bauto_incrementing_id\\b.*", Pattern.CASE_INSENSITIVE);
+
private static Pattern TABLE_NOT_FOUND =
Pattern.compile(".*\\bTable '(.*)' not found\\b.*", Pattern.CASE_INSENSITIVE);
@@ -66,6 +69,9 @@ public class UnsupportedChecker {
if (INPUT_FILE_NAME.matcher(s).matches() || FILE_POSITION.matcher(s).matches()) {
throw new UnsupportedFeatureException("Virtual columns not supported.");
}
+ if (AUTO_INCREMENTING_ID.matcher(s).matches()) {
+ throw new UnsupportedFeatureException("auto_incrementing_id not supported.");
+ }
}
public static void throwUnsupportedIfKnownException(Exception e,
diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_runtime_filter_with_timestamp_conversion.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_runtime_filter_with_timestamp_conversion.test
index e0eac06f8..280d6831c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu_runtime_filter_with_timestamp_conversion.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_runtime_filter_with_timestamp_conversion.test
@@ -27,7 +27,7 @@ INT,TIMESTAMP,INT,TIMESTAMP
10,2011-11-06 01:40:00,7,2011-11-06 01:40:00
10,2011-11-06 01:40:00,10,2011-11-06 01:40:00
---- RUNTIME_PROFILE
-row_regex: RF00.\[min_max\] <- from_utc_timestamp\(t2.ts, 'America/Los_Angeles'\)
+row_regex: RF00.\[min_max\] <- from_utc_timestamp\(.*.ts, 'America/Los_Angeles'\)
====
---- QUERY
# Test that kudu min_max runtime filter with timestamps can be work correctly in the
@@ -41,7 +41,7 @@ INT,BIGINT,TIMESTAMP,STRING
5,1320566400,2011-11-06 01:00:00,'2011-11-06 01:00:00'
8,1320570000,2011-11-06 01:00:00,'2011-11-06 01:00:00'
---- RUNTIME_PROFILE
-row_regex: RF00.\[min_max\] <- t2.ts
+row_regex: .*RF00.\[min_max\] .*\.ts
====
---- QUERY
# Test that kudu bloom runtime filter with timestamps can be work correctly in the
@@ -69,5 +69,5 @@ INT,TIMESTAMP,TIMESTAMP
9,2011-11-06 17:20:00,2011-11-06 09:20:00
10,2011-11-06 17:40:00,2011-11-06 09:40:00
---- RUNTIME_PROFILE
-row_regex: RF00.\[bloom\] <- utc_to_unix_micros\(to_utc_timestamp\(from_utc_timestamp\(t2.ts, 'Asia/Shanghai'\), 'Asia/Shanghai'\)\)
+row_regex: RF00.\[bloom\] <- utc_to_unix_micros\(to_utc_timestamp\(from_utc_timestamp\(.*.ts, 'Asia/Shanghai'\), 'Asia/Shanghai'\)\)
====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/partitioned-top-n-high-ndv.test b/testdata/workloads/functional-query/queries/QueryTest/partitioned-top-n-high-ndv.test
index 1da2ee52d..a69420e90 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/partitioned-top-n-high-ndv.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/partitioned-top-n-high-ndv.test
@@ -36,14 +36,14 @@ select * from vw1 where rk<=2 LIMIT 100;
'|'
'02:ANALYTIC'
'| functions: rank()'
-'| partition by: part1, part2, part3, col4, col5'
+row_regex:'| partition by: .*part1, .*part2, .*part3, .*col4, .*col5'
'| order by: col6 ASC'
'| window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW'
'| row-size=80B cardinality=9223372.04T'
'|'
'05:TOP-N'
'| partition by: part1, part2, part3, col4, col5'
-'| order by: col6 ASC'
+row_regex:'| order by: col6 ASC.*'
'| partition limit: 2 (include ties)'
'| row-size=72B cardinality=9223372.04T'
'|'
@@ -51,7 +51,7 @@ select * from vw1 where rk<=2 LIMIT 100;
'|'
'01:TOP-N'
'| partition by: part1, part2, part3, col4, col5'
-'| order by: col6 ASC'
+row_regex:'| order by: col6 ASC.*'
'| partition limit: 2 (include ties)'
'| source expr: rank() <= CAST(2 AS BIGINT)'
'| row-size=72B cardinality=9223372.04T'
diff --git a/testdata/workloads/functional-query/queries/QueryTest/runtime_filters.test b/testdata/workloads/functional-query/queries/QueryTest/runtime_filters.test
index c39b22779..912ea1b41 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/runtime_filters.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/runtime_filters.test
@@ -313,6 +313,10 @@ row_regex: 00:SCAN KUDU.*s[ ]+11.00K[ ]+11.00K
# join in its root, which produces filters for the scan of t1.
# Limits are added to force this plan, they do not affect results.
####################################################
+# IMPALA-15134: Calcite planner produces different cardinality. The runtime
+# filter exists for both plans, but the cardinality is not changing for
+# Calcite. The plans are mostly the same, though there is an extra SelectNode
+# in the Calcite plan.
set RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS;
set RUNTIME_FILTER_MODE=GLOBAL;
@@ -325,6 +329,8 @@ with t1 as (select month x, bigint_col y from alltypes limit 7301),
aggregation(SUM, Files rejected): 22
---- RUNTIME_PROFILE: table_format=kudu
row_regex: 00:SCAN KUDU.*s[ ]+620[ ]+1.82K
+---- CALCITE_PLANNER_RUNTIME_PROFILE: table_format=kudu
+row_regex: 00:SCAN KUDU.*s[ ]+620[ ]+7.30K
====
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index bc0996432..e5bf4a59d 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -1080,7 +1080,13 @@ class ImpalaTestSuite(BaseTestSuite):
.replace(INTERNAL_LISTEN_IP, '$INTERNAL_LISTEN_IP')
rt_profile_info = None
- if 'RUNTIME_PROFILE_%s' % table_format_info.file_format in test_section:
+ if IS_CALCITE_PLANNER and 'CALCITE_PLANNER_RUNTIME_PROFILE_%s' \
+ % table_format_info.file_format in test_section:
+ # If this table format has a RUNTIME_PROFILE section specifically for it,
+ # evaluate that section and ignore any general RUNTIME_PROFILE sections.
+ rt_profile_info = 'CALCITE_PLANNER_RUNTIME_PROFILE_%s' \
+ % table_format_info.file_format
+ elif 'RUNTIME_PROFILE_%s' % table_format_info.file_format in test_section:
# If this table format has a RUNTIME_PROFILE section specifically for it,
# evaluate that section and ignore any general RUNTIME_PROFILE sections.
rt_profile_info = 'RUNTIME_PROFILE_%s' % table_format_info.file_format
diff --git a/tests/util/test_file_parser.py b/tests/util/test_file_parser.py
index d7502deec..924932126 100644
--- a/tests/util/test_file_parser.py
+++ b/tests/util/test_file_parser.py
@@ -296,11 +296,13 @@ def parse_test_file_text(text, valid_section_names, skip_unknown_sections=True):
# table format. RUNTIME_PROFILE sections with a comment are only evaluated for the
# specified format. If there is a RUNTIME_PROFILE section without a comment, it is
# evaluated for all formats that don't have a commented section for this query.
- if subsection_name == 'RUNTIME_PROFILE':
+ if subsection_name == 'RUNTIME_PROFILE' or \
+ subsection_name == 'CALCITE_PLANNER_RUNTIME_PROFILE':
if subsection_comment:
table_formats = parse_runtime_profile_table_formats(subsection_comment)
for table_format in table_formats:
- subsection_name_for_format = 'RUNTIME_PROFILE_%s' % table_format
+ subsection_name_for_format = \
+ '%s_%s' % (subsection_name, table_format)
parsed_sections[subsection_name_for_format] = subsection_str
continue