You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2026/08/25 15:36:56 UTC

(impala) 02/02: IMPALA-15284: Cover the pre-conversion seam programs

This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 13e8b9618b00f3d3712e8d1767d1b3f98fbb489d
Author: Aleksandr Efimov <dz...@gmail.com>
AuthorDate: Sun Aug 23 17:28:18 2026 +0300

    IMPALA-15284: Cover the pre-conversion seam programs
    
    The direct Calcite-only harness pins the client-visible labels, the
    exact output types, the PERCENT_RANK rewrite and seven query shapes.
    Everything the seam runs in between is exercised only indirectly, so a
    change there surfaces as a golden-file diff elsewhere, or not at all.
    
    Widen the corpus to twenty-four shapes: conditional expressions, string
    and decimal functions, DISTINCT, HAVING, LIMIT with OFFSET, INTERSECT,
    EXCEPT, outer and cross joins, semi and anti joins from IN and NOT IN,
    EXISTS, a scalar subquery, an inline view, a common table expression
    and LEAD/LAG. The synthetic fixture gains the columns and the second
    table these need.
    
    Pin what the programs at the seam do to the plan: CUME_DIST and NTILE
    expansion next to the existing PERCENT_RANK case, nested projects
    merging into one, unused columns trimmed, multi-row VALUES arriving as
    a single Values, and the Impala decimal typing of FLOOR and CEIL
    including the cap at the maximum precision. Check the unsupported
    complex-type case on unnesting as well as on projection.
    
    Pin cte_threshold too: off by default, and above the threshold the
    repeated subtree becomes an ImpalaSequence over a producer and one
    consumer per reference. With the option set the seam stops being a
    plan of standard Calcite relations, which is worth stating in a test.
    
    Testing:
    - CalciteOptimizerTest: 11 tests, 0 failures, 0 errors on Linux
    - git diff --check
    
    Change-Id: I8a4403b5fbdd59e2eea79139891d8bb7c4a0413b
    Assisted-by: Claude Opus 5 (Anthropic)
    Reviewed-on: http://gerrit.cloudera.org:8080/24724
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
    Reviewed-by: Steve Carlin <sc...@cloudera.com>
    Reviewed-by: Michael Smith <mi...@cloudera.com>
---
 .../calcite/service/CalciteOptimizerTest.java      | 287 ++++++++++++++++++++-
 1 file changed, 277 insertions(+), 10 deletions(-)

diff --git a/java/calcite-planner/src/test/java/org/apache/impala/calcite/service/CalciteOptimizerTest.java b/java/calcite-planner/src/test/java/org/apache/impala/calcite/service/CalciteOptimizerTest.java
index 5fdae4e17..96e104572 100644
--- a/java/calcite-planner/src/test/java/org/apache/impala/calcite/service/CalciteOptimizerTest.java
+++ b/java/calcite-planner/src/test/java/org/apache/impala/calcite/service/CalciteOptimizerTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -30,15 +31,20 @@ import java.util.Set;
 import com.google.common.collect.ImmutableList;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.RelVisitor;
+import org.apache.calcite.rel.logical.LogicalFilter;
 import org.apache.calcite.rel.logical.LogicalProject;
 import org.apache.calcite.rel.logical.LogicalTableScan;
+import org.apache.calcite.rel.logical.LogicalValues;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rex.RexOver;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexShuttle;
 import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.impala.calcite.rel.node.ImpalaCTEConsumer;
+import org.apache.impala.calcite.rel.node.ImpalaCTEProducer;
 import org.apache.impala.calcite.rel.node.ImpalaPlanRel;
+import org.apache.impala.calcite.rel.node.ImpalaSequence;
 import org.apache.impala.common.FrontendTestBase;
 import org.apache.impala.common.ImpalaException;
 import org.apache.impala.common.UnsupportedFeatureException;
@@ -51,6 +57,7 @@ public class CalciteOptimizerTest extends FrontendTestBase {
   private static final String TEST_DB = "calcite_optimizer_test";
   private static final String TEST_TABLE = TEST_DB + ".alltypes";
   private static final String TEST_COMPLEX_TABLE = TEST_DB + ".complex_types";
+  private static final String TEST_DIM_TABLE = TEST_DB + ".dim";
 
   private static final List<SupportedQuery> SUPPORTED_QUERIES = ImmutableList.of(
       new SupportedQuery("values",
@@ -79,13 +86,91 @@ public class CalciteOptimizerTest extends FrontendTestBase {
       new SupportedQuery("analytic",
           "select id, row_number() over (order by int_col) as row_num "
               + "from " + TEST_TABLE,
-          "id", "row_num"));
+          "id", "row_num"),
+      new SupportedQuery("analytic_lead_lag",
+          "select lead(int_col, 1) over (order by id) as next_int, "
+              + "lag(int_col, 1) over (order by id) as previous_int "
+              + "from " + TEST_TABLE,
+          "next_int", "previous_int"),
+      new SupportedQuery("conditional",
+          "select case when int_col > 1 then 'big' else 'small' end as bucket, "
+              + "coalesce(int_col, 0) as maybe_int "
+              + "from " + TEST_TABLE,
+          "bucket", "maybe_int"),
+      new SupportedQuery("string_functions",
+          "select concat(string_col, 'x') as concatenated, "
+              + "length(string_col) as string_length "
+              + "from " + TEST_TABLE,
+          "concatenated", "string_length"),
+      new SupportedQuery("decimal_arithmetic",
+          "select decimal_col * cast(2 as decimal(3, 1)) as scaled "
+              + "from " + TEST_TABLE,
+          "scaled"),
+      new SupportedQuery("distinct",
+          "select distinct int_col as distinct_int from " + TEST_TABLE,
+          "distinct_int"),
+      new SupportedQuery("having",
+          "select int_col as group_key, count(*) as row_count "
+              + "from " + TEST_TABLE + " group by int_col having count(*) > 1",
+          "group_key", "row_count"),
+      new SupportedQuery("sort_with_offset",
+          "select int_col as sorted_int from " + TEST_TABLE + " "
+              + "order by int_col limit 5 offset 3",
+          "sorted_int"),
+      new SupportedQuery("intersect",
+          "select int_col as intersected_int from " + TEST_TABLE + " "
+              + "intersect select int_col from " + TEST_TABLE,
+          "intersected_int"),
+      new SupportedQuery("except",
+          "select int_col as remaining_int from " + TEST_TABLE + " "
+              + "except select int_col from " + TEST_TABLE,
+          "remaining_int"),
+      new SupportedQuery("outer_join",
+          "select l.id as left_id, r.name as right_name "
+              + "from " + TEST_TABLE + " l left outer join " + TEST_DIM_TABLE + " r "
+              + "on l.id = r.id",
+          "left_id", "right_name"),
+      new SupportedQuery("cross_join",
+          "select l.id as left_id, r.name as right_name "
+              + "from " + TEST_TABLE + " l, " + TEST_DIM_TABLE + " r",
+          "left_id", "right_name"),
+      new SupportedQuery("semi_join",
+          "select id as semi_id from " + TEST_TABLE + " "
+              + "where id in (select id from " + TEST_DIM_TABLE + ")",
+          "semi_id"),
+      new SupportedQuery("anti_join",
+          "select id as anti_id from " + TEST_TABLE + " "
+              + "where id not in (select id from " + TEST_DIM_TABLE + ")",
+          "anti_id"),
+      new SupportedQuery("exists_subquery",
+          "select l.id as exists_id from " + TEST_TABLE + " l "
+              + "where exists (select 1 from " + TEST_DIM_TABLE + " r "
+              + "where r.id = l.id)",
+          "exists_id"),
+      new SupportedQuery("scalar_subquery",
+          "select id as scalar_id, "
+              + "(select count(*) from " + TEST_DIM_TABLE + ") as dim_rows "
+              + "from " + TEST_TABLE,
+          "scalar_id", "dim_rows"),
+      new SupportedQuery("inline_view",
+          "select v.projected_int from "
+              + "(select int_col as projected_int from " + TEST_TABLE + ") v "
+              + "where v.projected_int > 2",
+          "projected_int"),
+      new SupportedQuery("common_table_expression",
+          "with c as (select int_col as cte_int from " + TEST_TABLE + ") "
+              + "select cte_int from c",
+          "cte_int"));
 
   @Before
   public void addSyntheticTestTable() {
     addTestDb(TEST_DB, "Synthetic Calcite optimizer test database");
     addTestTable("create table " + TEST_TABLE
-        + " (id int, int_col int, bigint_col bigint) stored as parquet");
+        + " (id int, int_col int, bigint_col bigint, string_col string, "
+        + "decimal_col decimal(9, 2), wide_decimal_col decimal(38, 2)) "
+        + "stored as parquet");
+    addTestTable("create table " + TEST_DIM_TABLE
+        + " (id int, name string) stored as parquet");
     addTestTable("create table " + TEST_COMPLEX_TABLE
         + " (id int, int_array_col array<int>) stored as parquet");
   }
@@ -146,6 +231,141 @@ public class CalciteOptimizerTest extends FrontendTestBase {
     assertEquals(result.outputLabels(), optimizationResult.outputLabels());
   }
 
+  @Test
+  public void testCumeDistAndNtileAreRewrittenBeforeLogicalSeam()
+      throws ImpalaException {
+    PreImpalaConvertResult cumeDist = createPreImpalaConvertPlan(
+        "select cume_dist() over (order by int_col) as cd from " + TEST_TABLE);
+
+    assertEquals(ImmutableList.of("cd"), cumeDist.outputLabels());
+    assertEquals(SqlTypeName.DOUBLE,
+        cumeDist.outputTypes().get(0).getSqlTypeName());
+    Set<String> cumeDistFunctions = collectOverFunctions(cumeDist.plan());
+    assertFalse(cumeDistFunctions.contains("cume_dist"));
+    assertTrue(cumeDistFunctions.contains("rank"));
+    assertTrue(cumeDistFunctions.contains("count"));
+
+    PreImpalaConvertResult ntile = createPreImpalaConvertPlan(
+        "select ntile(4) over (order by int_col) as tile from " + TEST_TABLE);
+
+    assertEquals(ImmutableList.of("tile"), ntile.outputLabels());
+    assertEquals(SqlTypeName.BIGINT, ntile.outputTypes().get(0).getSqlTypeName());
+    Set<String> ntileFunctions = collectOverFunctions(ntile.plan());
+    assertFalse(ntileFunctions.contains("ntile"));
+    assertTrue(ntileFunctions.contains("row_number"));
+    assertTrue(ntileFunctions.contains("count"));
+  }
+
+  @Test
+  public void testNestedProjectsAreMergedBeforeLogicalSeam() throws ImpalaException {
+    PreImpalaConvertResult result = createPreImpalaConvertPlan(
+        "select projected_int + 1 as shifted_int from "
+            + "(select int_col as projected_int from " + TEST_TABLE + ") v");
+
+    assertEquals(ImmutableList.of("shifted_int"), result.outputLabels());
+    assertEquals(SqlTypeName.BIGINT, result.outputTypes().get(0).getSqlTypeName());
+    // PROJECT_MERGE runs last in the pre-conversion program, so the inner and
+    // outer projects reach the seam as a single one.
+    assertEquals(ImmutableList.of(LogicalProject.class, LogicalTableScan.class),
+        collectRelNodeHierarchy(result.plan()));
+  }
+
+  @Test
+  public void testFloorAndCeilCarryImpalaDecimalTypeToLogicalSeam()
+      throws ImpalaException {
+    for (String function : ImmutableList.of("floor", "ceil")) {
+      // Impala infers decimal(p + 1, 0) for these, capped at the maximum
+      // precision, which is why they are not plain Calcite operators.
+      RelDataType narrow = createPreImpalaConvertPlan(
+          "select " + function + "(decimal_col) as rounded from " + TEST_TABLE)
+          .outputTypes().get(0);
+      assertEquals(function, SqlTypeName.DECIMAL, narrow.getSqlTypeName());
+      assertEquals(function, 10, narrow.getPrecision());
+      assertEquals(function, 0, narrow.getScale());
+
+      RelDataType wide = createPreImpalaConvertPlan(
+          "select " + function + "(wide_decimal_col) as rounded from " + TEST_TABLE)
+          .outputTypes().get(0);
+      assertEquals(function, SqlTypeName.DECIMAL, wide.getSqlTypeName());
+      assertEquals(function, 38, wide.getPrecision());
+      assertEquals(function, 0, wide.getScale());
+    }
+  }
+
+  @Test
+  public void testClientLabelsSurviveARenamingProjection() throws ImpalaException {
+    // The labels come from the validated statement, not from the field names of
+    // the plan, which the pre-conversion rules are free to rewrite.
+    PreImpalaConvertResult result = createPreImpalaConvertPlan(
+        "select v.projected_int as client_label from "
+            + "(select int_col as projected_int from " + TEST_TABLE + ") v "
+            + "where v.projected_int > 5");
+
+    assertEquals(ImmutableList.of("client_label"), result.outputLabels());
+    assertEquals(1, result.outputTypes().size());
+    assertEquals(SqlTypeName.INTEGER, result.outputTypes().get(0).getSqlTypeName());
+  }
+
+  @Test
+  public void testUnusedColumnsAreTrimmedBeforeLogicalSeam() throws ImpalaException {
+    PreImpalaConvertResult result = createPreImpalaConvertPlan(
+        "select v.projected_int from (select int_col as projected_int, "
+            + "bigint_col as unused_long from " + TEST_TABLE + ") v "
+            + "where v.projected_int > 5");
+
+    assertEquals(ImmutableList.of(LogicalFilter.class, LogicalProject.class,
+        LogicalTableScan.class), collectRelNodeHierarchy(result.plan()));
+    // The inline view selects two columns and the statement uses one, so the
+    // field trimmer is what decides how wide the projection at the seam is.
+    assertEquals(1, findOnly(result.plan(), LogicalProject.class).getProjects().size());
+  }
+
+  @Test
+  public void testValuesRowsAreCombinedBeforeLogicalSeam() throws ImpalaException {
+    PreImpalaConvertResult result = createPreImpalaConvertPlan(
+        "select * from (values (1, 'a'), (2, 'b')) as v(number_col, text_col)");
+
+    assertEquals(ImmutableList.of("number_col", "text_col"), result.outputLabels());
+    // Each row starts as its own relation; they reach the seam as one Values.
+    assertEquals(ImmutableList.of(LogicalValues.class),
+        collectRelNodeHierarchy(result.plan()));
+    assertEquals(2, findOnly(result.plan(), LogicalValues.class).getTuples().size());
+  }
+
+  @Test
+  public void testCteThresholdMaterializesRepeatedSubtreesAtLogicalSeam()
+      throws ImpalaException {
+    String twoUses = "with c as (select id, int_col from " + TEST_TABLE + " "
+        + "where int_col > 1) select a.id as left_id, b.int_col as right_int "
+        + "from c a join c b on a.id = b.id";
+    String threeUses = "with c as (select id, int_col from " + TEST_TABLE + " "
+        + "where int_col > 1) select a.id as left_id from c a "
+        + "join c b on a.id = b.id join c d on a.id = d.id";
+
+    // CTE planning is off by default, so the seam sees the expanded plan.
+    assertTrue(collectCteRels(createPreImpalaConvertPlan(twoUses).plan()).isEmpty());
+
+    // With cte_threshold set, the repeated subtree becomes a producer, one
+    // consumer per use, and a sequence at the root. Those are Impala relations,
+    // so the seam stops being a plan of standard Calcite nodes.
+    assertEquals(
+        ImmutableList.of(ImpalaSequence.class, ImpalaCTEConsumer.class,
+            ImpalaCTEConsumer.class, ImpalaCTEProducer.class),
+        collectCteRels(
+            createPreImpalaConvertPlan(twoUses, cteThreshold(1)).plan()));
+
+    // The option counts references: two uses stay expanded below a threshold of
+    // two, three uses do not.
+    assertTrue(collectCteRels(
+        createPreImpalaConvertPlan(twoUses, cteThreshold(2)).plan()).isEmpty());
+    assertEquals(
+        ImmutableList.of(ImpalaSequence.class, ImpalaCTEConsumer.class,
+            ImpalaCTEConsumer.class, ImpalaCTEConsumer.class,
+            ImpalaCTEProducer.class),
+        collectCteRels(
+            createPreImpalaConvertPlan(threeUses, cteThreshold(2)).plan()));
+  }
+
   @Test
   public void testSupportedQueryCorpusReachesImpalaPlan() throws ImpalaException {
     for (SupportedQuery query : SUPPORTED_QUERIES) {
@@ -166,12 +386,17 @@ public class CalciteOptimizerTest extends FrontendTestBase {
 
   @Test
   public void testUnsupportedQueryIsNotHiddenByFallback() throws ImpalaException {
-    try {
-      optimizeWithCalciteOnly(
-          "select int_array_col from " + TEST_COMPLEX_TABLE);
-      fail("Expected complex type projection to remain explicitly unsupported");
-    } catch (UnsupportedFeatureException expected) {
-      // The direct harness must preserve the classification rather than invoke fallback.
+    List<String> unsupportedQueries = ImmutableList.of(
+        "select int_array_col from " + TEST_COMPLEX_TABLE,
+        "select a.item from " + TEST_COMPLEX_TABLE + " t, t.int_array_col a");
+    for (String sql : unsupportedQueries) {
+      try {
+        optimizeWithCalciteOnly(sql);
+        fail("Expected complex types to remain explicitly unsupported: " + sql);
+      } catch (UnsupportedFeatureException expected) {
+        // The direct harness must preserve the classification rather than invoke
+        // fallback.
+      }
     }
   }
 
@@ -182,7 +407,12 @@ public class CalciteOptimizerTest extends FrontendTestBase {
    */
   private PreImpalaConvertResult createPreImpalaConvertPlan(String sql)
       throws ImpalaException {
-    CalcitePlanInput input = createCalcitePlanInput(sql);
+    return createPreImpalaConvertPlan(sql, new TQueryOptions());
+  }
+
+  private PreImpalaConvertResult createPreImpalaConvertPlan(String sql,
+      TQueryOptions queryOptions) throws ImpalaException {
+    CalcitePlanInput input = createCalcitePlanInput(sql, queryOptions);
     RelNode plan = input.optimizer().createPreImpalaConvertPlan(input.logicalPlan());
     return new PreImpalaConvertResult(
         plan, input.outputLabels(), collectOutputTypes(plan));
@@ -196,6 +426,11 @@ public class CalciteOptimizerTest extends FrontendTestBase {
   }
 
   private CalcitePlanInput createCalcitePlanInput(String sql) throws ImpalaException {
+    return createCalcitePlanInput(sql, new TQueryOptions());
+  }
+
+  private CalcitePlanInput createCalcitePlanInput(String sql,
+      TQueryOptions queryOptions) throws ImpalaException {
     CalciteAnalysisResult analysisResult = (CalciteAnalysisResult) parseAndAnalyze(
         sql, feFixture_.createAnalysisCtx(), new CalciteCompilerFactory());
     CalciteRelNodeConverter relNodeConverter =
@@ -203,7 +438,7 @@ public class CalciteOptimizerTest extends FrontendTestBase {
     RelNode logicalPlan =
         relNodeConverter.convert(analysisResult.getValidatedNode());
     CalciteOptimizer optimizer = new CalciteOptimizer(
-        analysisResult, NoOpEventSequence.INSTANCE, new TQueryOptions());
+        analysisResult, NoOpEventSequence.INSTANCE, queryOptions);
     ImmutableList<String> outputLabels = ImmutableList.copyOf(
         relNodeConverter.getFieldNames(analysisResult.getValidatedNode()));
     return new CalcitePlanInput(optimizer, logicalPlan, outputLabels);
@@ -265,6 +500,38 @@ public class CalciteOptimizerTest extends FrontendTestBase {
     return functions;
   }
 
+  private static TQueryOptions cteThreshold(int referenceThreshold) {
+    TQueryOptions queryOptions = new TQueryOptions();
+    queryOptions.setCte_threshold(referenceThreshold);
+    return queryOptions;
+  }
+
+  private static List<Class<? extends RelNode>> collectCteRels(RelNode plan) {
+    ImmutableList.Builder<Class<? extends RelNode>> cteRels = ImmutableList.builder();
+    for (Class<? extends RelNode> relClass : collectRelNodeHierarchy(plan)) {
+      if (relClass.equals(ImpalaSequence.class)
+          || relClass.equals(ImpalaCTEProducer.class)
+          || relClass.equals(ImpalaCTEConsumer.class)) {
+        cteRels.add(relClass);
+      }
+    }
+    return cteRels.build();
+  }
+
+  private static <T extends RelNode> T findOnly(RelNode plan, Class<T> relClass) {
+    List<T> matches = new ArrayList<>();
+    new RelVisitor() {
+      @Override
+      public void visit(RelNode node, int ordinal, RelNode parent) {
+        if (relClass.isInstance(node)) matches.add(relClass.cast(node));
+        super.visit(node, ordinal, parent);
+      }
+    }.go(plan);
+    assertEquals("Unexpected number of " + relClass.getSimpleName() + " nodes",
+        1, matches.size());
+    return matches.get(0);
+  }
+
   private static List<Class<? extends RelNode>> collectRelNodeHierarchy(RelNode plan) {
     ImmutableList.Builder<Class<? extends RelNode>> hierarchy =
         ImmutableList.builder();