You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2023/08/21 01:03:40 UTC

[calcite] branch main updated (dc3f11bf9b -> 0d3456705a)

This is an automated email from the ASF dual-hosted git repository.

jhyde pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git


    from dc3f11bf9b [CALCITE-985] Validate MERGE
     new 357f06bb85 In Puffin, allow an action to test whether it is looking at the last line of a source
     new 0d3456705a [CALCITE-5941] Support LITERAL_AGG in Interpreter

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../apache/calcite/interpreter/AggregateNode.java  | 174 ++++++++++++---------
 .../main/java/org/apache/calcite/util/Puffin.java  |  46 ++++--
 .../org/apache/calcite/test/InterpreterTest.java   |  27 +++-
 .../java/org/apache/calcite/test/PuffinTest.java   |  13 +-
 .../apache/calcite/test/ScannableTableTest.java    | 128 ++++++++-------
 5 files changed, 233 insertions(+), 155 deletions(-)


[calcite] 02/02: [CALCITE-5941] Support LITERAL_AGG in Interpreter

Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jhyde pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git

commit 0d3456705a13e26e73f51a7ea978e817a1d00154
Author: Julian Hyde <jh...@apache.org>
AuthorDate: Sun Aug 20 00:36:39 2023 -0700

    [CALCITE-5941] Support LITERAL_AGG in Interpreter
    
    Also clean up ScannableTableTest, declaring the hitherto hidden third
    column of the "beatles" table.
---
 .../apache/calcite/interpreter/AggregateNode.java  | 174 ++++++++++++---------
 .../org/apache/calcite/test/InterpreterTest.java   |  27 +++-
 .../apache/calcite/test/ScannableTableTest.java    | 128 ++++++++-------
 3 files changed, 186 insertions(+), 143 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java
index 9f9db72dcc..e461b238de 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java
@@ -38,6 +38,8 @@ import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.runtime.FunctionContexts;
 import org.apache.calcite.schema.FunctionContext;
 import org.apache.calcite.schema.impl.AggregateFunctionImpl;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.fun.SqlInternalOperators;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.validate.SqlConformance;
 import org.apache.calcite.sql.validate.SqlConformanceEnum;
@@ -91,7 +93,8 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
     ImmutableList.Builder<AccumulatorFactory> builder = ImmutableList.builder();
     for (AggregateCall aggregateCall : rel.getAggCallList()) {
       @SuppressWarnings("method.invocation.invalid")
-      AccumulatorFactory accumulator = getAccumulator(aggregateCall, false);
+      AccumulatorFactory accumulator =
+          getAccumulator(compiler, aggregateCall, false);
       builder.add(accumulator);
     }
     accumulatorFactories = builder.build();
@@ -110,91 +113,32 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
     }
   }
 
-  private AccumulatorFactory getAccumulator(final AggregateCall call,
-      boolean ignoreFilter) {
+  private AccumulatorFactory getAccumulator(Compiler compiler,
+      final AggregateCall call, boolean ignoreFilter) {
     if (call.filterArg >= 0 && !ignoreFilter) {
-      final AccumulatorFactory factory = getAccumulator(call, true);
+      final AccumulatorFactory factory = getAccumulator(compiler, call, true);
       return () -> {
         final Accumulator accumulator = factory.get();
         return new FilterAccumulator(accumulator, call.filterArg);
       };
     }
-    if (call.getAggregation() == SqlStdOperatorTable.COUNT) {
+    final SqlAggFunction op = call.getAggregation();
+    if (op == SqlStdOperatorTable.COUNT) {
       return () -> new CountAccumulator(call);
-    } else if (call.getAggregation() == SqlStdOperatorTable.SUM
-        || call.getAggregation() == SqlStdOperatorTable.SUM0) {
-      final Class<?> clazz;
-      switch (call.type.getSqlTypeName()) {
-      case DOUBLE:
-      case REAL:
-      case FLOAT:
-        clazz = DoubleSum.class;
-        break;
-      case DECIMAL:
-        clazz = BigDecimalSum.class;
-        break;
-      case INTEGER:
-        clazz = IntSum.class;
-        break;
-      case BIGINT:
-      default:
-        clazz = LongSum.class;
-        break;
-      }
-      if (call.getAggregation() == SqlStdOperatorTable.SUM) {
-        return new UdaAccumulatorFactory(getAggFunction(clazz), call, true,
-            dataContext);
-      } else {
-        return new UdaAccumulatorFactory(getAggFunction(clazz), call, false,
-            dataContext);
-      }
-    } else if (call.getAggregation() == SqlStdOperatorTable.MIN) {
-      final Class<?> clazz;
-      switch (call.getType().getSqlTypeName()) {
-      case INTEGER:
-        clazz = MinInt.class;
-        break;
-      case FLOAT:
-        clazz = MinFloat.class;
-        break;
-      case DOUBLE:
-      case REAL:
-        clazz = MinDouble.class;
-        break;
-      case DECIMAL:
-        clazz = MinBigDecimal.class;
-        break;
-      case BOOLEAN:
-        clazz = MinBoolean.class;
-        break;
-      default:
-        clazz = MinLong.class;
-        break;
-      }
-      return new UdaAccumulatorFactory(getAggFunction(clazz), call, true,
-          dataContext);
-    } else if (call.getAggregation() == SqlStdOperatorTable.MAX) {
-      final Class<?> clazz;
-      switch (call.getType().getSqlTypeName()) {
-      case INTEGER:
-        clazz = MaxInt.class;
-        break;
-      case FLOAT:
-        clazz = MaxFloat.class;
-        break;
-      case DOUBLE:
-      case REAL:
-        clazz = MaxDouble.class;
-        break;
-      case DECIMAL:
-        clazz = MaxBigDecimal.class;
-        break;
-      default:
-        clazz = MaxLong.class;
-        break;
-      }
+    } else if (op == SqlStdOperatorTable.SUM
+        || op == SqlStdOperatorTable.SUM0) {
+      final Class<?> clazz = sumClass(call);
+      return new UdaAccumulatorFactory(getAggFunction(clazz), call,
+          op == SqlStdOperatorTable.SUM, dataContext);
+    } else if (op == SqlStdOperatorTable.MAX
+        || op == SqlStdOperatorTable.MIN) {
+      final Class<?> clazz = maxMinClass(call);
       return new UdaAccumulatorFactory(getAggFunction(clazz), call, true,
           dataContext);
+    } else if (op == SqlInternalOperators.LITERAL_AGG) {
+      final Scalar scalar = compiler.compile(call.rexList, null);
+      final Object value = scalar.execute(compiler.createContext());
+      return () -> new LiteralAccumulator(value);
     } else {
       final JavaTypeFactory typeFactory =
           (JavaTypeFactory) rel.getCluster().getTypeFactory();
@@ -269,6 +213,41 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
     }
   }
 
+  private static Class<?> maxMinClass(AggregateCall call) {
+    boolean max = call.getAggregation() == SqlStdOperatorTable.MAX;
+    switch (call.getType().getSqlTypeName()) {
+    case INTEGER:
+      return max ? MaxInt.class : MinInt.class;
+    case FLOAT:
+      return max ? MaxFloat.class : MinFloat.class;
+    case DOUBLE:
+    case REAL:
+      return max ? MaxDouble.class : MinDouble.class;
+    case DECIMAL:
+      return max ? MaxBigDecimal.class : MinBigDecimal.class;
+    case BOOLEAN:
+      return max ? MaxBoolean.class : MinBoolean.class;
+    default:
+      return max ? MaxLong.class : MinLong.class;
+    }
+  }
+
+  private static Class<?> sumClass(AggregateCall call) {
+    switch (call.type.getSqlTypeName()) {
+    case DOUBLE:
+    case REAL:
+    case FLOAT:
+      return DoubleSum.class;
+    case DECIMAL:
+      return BigDecimalSum.class;
+    case INTEGER:
+      return IntSum.class;
+    case BIGINT:
+    default:
+      return LongSum.class;
+    }
+  }
+
   private static AggregateFunctionImpl getAggFunction(Class<?> clazz) {
     return requireNonNull(
         AggregateFunctionImpl.create(clazz),
@@ -303,6 +282,22 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
     }
   }
 
+  /** Accumulator for calls to the LITERAL_AGG function. */
+  private static class LiteralAccumulator implements Accumulator {
+    private final @Nullable Object value;
+
+    LiteralAccumulator(@Nullable Object value) {
+      this.value = value;
+    }
+
+    @Override public void send(Row row) {
+    }
+
+    @Override public @Nullable Object end() {
+      return value;
+    }
+  }
+
   /** Creates an {@link Accumulator}. */
   private interface AccumulatorFactory extends Supplier<Accumulator> {
   }
@@ -616,7 +611,7 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
     }
 
     public Boolean add(Boolean accumulator, Boolean value) {
-      return accumulator.compareTo(value) < 0 ? accumulator : value;
+      return accumulator && value;
     }
 
     public Boolean merge(Boolean accumulator0, Boolean accumulator1) {
@@ -677,6 +672,29 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
     }
   }
 
+  /** Implementation of {@code MAX} function to calculate the maximum of
+   * {@code boolean} values as a user-defined aggregate.
+   */
+  public static class MaxBoolean {
+    public MaxBoolean() { }
+
+    public Boolean init() {
+      return Boolean.FALSE;
+    }
+
+    public Boolean add(Boolean accumulator, Boolean value) {
+      return accumulator || value;
+    }
+
+    public Boolean merge(Boolean accumulator0, Boolean accumulator1) {
+      return add(accumulator0, accumulator1);
+    }
+
+    public Boolean result(Boolean accumulator) {
+      return accumulator;
+    }
+  }
+
   /** Accumulator factory based on a user-defined aggregate function. */
   private static class UdaAccumulatorFactory implements AccumulatorFactory {
     final AggregateFunctionImpl aggFunction;
diff --git a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java
index f7e428e4bb..a98a19625b 100644
--- a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java
+++ b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java
@@ -292,7 +292,8 @@ class InterpreterTest {
   @Test void testInterpretScannableTable() {
     rootSchema.add("beatles", new ScannableTableTest.BeatlesTable());
     sql("select * from \"beatles\" order by \"i\"")
-        .returnsRows("[4, John]", "[4, Paul]", "[5, Ringo]", "[6, George]");
+        .returnsRows("[4, John, 1940]", "[4, Paul, 1942]", "[5, Ringo, 1940]",
+            "[6, George, 1943]");
   }
 
   /** Tests executing a plan on a
@@ -349,6 +350,22 @@ class InterpreterTest {
             "[Ringo, 1]");
   }
 
+  /** Tests a GROUP BY query that uses
+   * {@link org.apache.calcite.sql.fun.SqlInternalOperators#LITERAL_AGG}. */
+  @Test void testAggregateLiteralAgg() {
+    rootSchema.add("beatles", new ScannableTableTest.BeatlesTable());
+    final Function<RelBuilder, RelNode> relFn =
+        b -> b.scan("beatles")
+            .aggregate(b.groupKey("k"),
+                b.count().as("c"),
+                b.literalAgg(true).as("t"),
+                b.literalAgg(-3).as("minus3"))
+            .build();
+    fixture().withRel(relFn)
+        .returnsRows("[1940, 2, true, -3]", "[1942, 1, true, -3]",
+            "[1943, 1, true, -3]");
+  }
+
   /** Tests executing a plan on a single-column
    * {@link org.apache.calcite.schema.ScannableTable} using an interpreter. */
   @Test void testInterpretSimpleScannableTable() {
@@ -730,11 +747,11 @@ class InterpreterTest {
   /** Tests projecting zero fields. */
   @Test void testZeroFields() {
     final List<RexLiteral> row = ImmutableList.of();
-    fixture()
-        .withRel(b ->
-            b.values(ImmutableList.of(row, row),
+    final Function<RelBuilder, RelNode> relFn =
+        b -> b.values(ImmutableList.of(row, row),
                 b.getTypeFactory().builder().build())
-                .build())
+            .build();
+    fixture().withRel(relFn)
         .returnsRows("[]", "[]");
   }
 }
diff --git a/core/src/test/java/org/apache/calcite/test/ScannableTableTest.java b/core/src/test/java/org/apache/calcite/test/ScannableTableTest.java
index 747e69555a..ee0e54919d 100644
--- a/core/src/test/java/org/apache/calcite/test/ScannableTableTest.java
+++ b/core/src/test/java/org/apache/calcite/test/ScannableTableTest.java
@@ -28,6 +28,7 @@ import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.runtime.Hook;
+import org.apache.calcite.runtime.PairList;
 import org.apache.calcite.schema.FilterableTable;
 import org.apache.calcite.schema.ProjectableFilterableTable;
 import org.apache.calcite.schema.ScannableTable;
@@ -67,52 +68,55 @@ import static org.hamcrest.Matchers.hasToString;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Unit test for {@link org.apache.calcite.schema.ScannableTable}.
  */
 public class ScannableTableTest {
-  @Test void testTens() throws SQLException {
-    final Enumerator<Object[]> cursor = tens();
-    assertTrue(cursor.moveNext());
-    assertThat(cursor.current()[0], equalTo((Object) 0));
-    assertThat(cursor.current().length, equalTo(1));
-    assertTrue(cursor.moveNext());
-    assertThat(cursor.current()[0], equalTo((Object) 10));
-    assertTrue(cursor.moveNext());
-    assertThat(cursor.current()[0], equalTo((Object) 20));
-    assertTrue(cursor.moveNext());
-    assertThat(cursor.current()[0], equalTo((Object) 30));
-    assertFalse(cursor.moveNext());
+  @Test void testTens() {
+    try (Enumerator<Object[]> cursor = tens()) {
+      assertTrue(cursor.moveNext());
+      assertThat(cursor.current()[0], equalTo(0));
+      assertThat(cursor.current().length, equalTo(1));
+      assertTrue(cursor.moveNext());
+      assertThat(cursor.current()[0], equalTo(10));
+      assertTrue(cursor.moveNext());
+      assertThat(cursor.current()[0], equalTo(20));
+      assertTrue(cursor.moveNext());
+      assertThat(cursor.current()[0], equalTo(30));
+      assertFalse(cursor.moveNext());
+    }
   }
 
   /** A table with one column. */
-  @Test void testSimple() throws Exception {
+  @Test void testSimple() {
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("simple", new SimpleTable())))
+        .with(newSchema("s", PairList.of("simple", new SimpleTable())))
         .query("select * from \"s\".\"simple\"")
         .returnsUnordered("i=0", "i=10", "i=20", "i=30");
   }
 
   /** A table with two columns. */
-  @Test void testSimple2() throws Exception {
+  @Test void testSimple2() {
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles", new BeatlesTable())))
+        .with(newSchema("s", PairList.of("beatles", new BeatlesTable())))
         .query("select * from \"s\".\"beatles\"")
-        .returnsUnordered("i=4; j=John",
-            "i=4; j=Paul",
-            "i=6; j=George",
-            "i=5; j=Ringo");
+        .returnsUnordered("i=4; j=John; k=1940",
+            "i=4; j=Paul; k=1942",
+            "i=6; j=George; k=1943",
+            "i=5; j=Ringo; k=1940");
   }
 
   /** A filter on a {@link FilterableTable} with two columns (cooperative). */
-  @Test void testFilterableTableCooperative() throws Exception {
+  @Test void testFilterableTableCooperative() {
     final StringBuilder buf = new StringBuilder();
     final Table table = new BeatlesFilterableTable(buf, true);
     final String explain = "PLAN="
         + "EnumerableInterpreter\n"
         + "  BindableTableScan(table=[[s, beatles]], filters=[[=($0, 4)]])";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles", table)))
+        .with(newSchema("s", PairList.of("beatles", table)))
         .query("select * from \"s\".\"beatles\" where \"i\" = 4")
         .explainContains(explain)
         .returnsUnordered("i=4; j=John; k=1940",
@@ -123,14 +127,14 @@ public class ScannableTableTest {
   }
 
   /** A filter on a {@link FilterableTable} with two columns (noncooperative). */
-  @Test void testFilterableTableNonCooperative() throws Exception {
+  @Test void testFilterableTableNonCooperative() {
     final StringBuilder buf = new StringBuilder();
     final Table table = new BeatlesFilterableTable(buf, false);
     final String explain = "PLAN="
         + "EnumerableInterpreter\n"
         + "  BindableTableScan(table=[[s, beatles2]], filters=[[=($0, 4)]])";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles2", table)))
+        .with(newSchema("s", PairList.of("beatles2", table)))
         .query("select * from \"s\".\"beatles2\" where \"i\" = 4")
         .explainContains(explain)
         .returnsUnordered("i=4; j=John; k=1940",
@@ -140,14 +144,14 @@ public class ScannableTableTest {
 
   /** A filter on a {@link org.apache.calcite.schema.ProjectableFilterableTable}
    * with two columns (cooperative). */
-  @Test void testProjectableFilterableCooperative() throws Exception {
+  @Test void testProjectableFilterableCooperative() {
     final StringBuilder buf = new StringBuilder();
     final Table table = new BeatlesProjectableFilterableTable(buf, true);
     final String explain = "PLAN="
         + "EnumerableInterpreter\n"
         + "  BindableTableScan(table=[[s, beatles]], filters=[[=($0, 4)]], projects=[[1]])";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles", table)))
+        .with(newSchema("s", PairList.of("beatles", table)))
         .query("select \"j\" from \"s\".\"beatles\" where \"i\" = 4")
         .explainContains(explain)
         .returnsUnordered("j=John",
@@ -158,14 +162,14 @@ public class ScannableTableTest {
         hasToString("returnCount=2, filter=<0, 4>, projects=[1, 0]"));
   }
 
-  @Test void testProjectableFilterableNonCooperative() throws Exception {
+  @Test void testProjectableFilterableNonCooperative() {
     final StringBuilder buf = new StringBuilder();
     final Table table = new BeatlesProjectableFilterableTable(buf, false);
     final String explain = "PLAN="
         + "EnumerableInterpreter\n"
         + "  BindableTableScan(table=[[s, beatles2]], filters=[[=($0, 4)]], projects=[[1]]";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles2", table)))
+        .with(newSchema("s", PairList.of("beatles2", table)))
         .query("select \"j\" from \"s\".\"beatles2\" where \"i\" = 4")
         .explainContains(explain)
         .returnsUnordered("j=John",
@@ -175,14 +179,14 @@ public class ScannableTableTest {
 
   /** A filter on a {@link org.apache.calcite.schema.ProjectableFilterableTable}
    * with two columns, and a project in the query. (Cooperative)*/
-  @Test void testProjectableFilterableWithProjectAndFilter() throws Exception {
+  @Test void testProjectableFilterableWithProjectAndFilter() {
     final StringBuilder buf = new StringBuilder();
     final Table table = new BeatlesProjectableFilterableTable(buf, true);
     final String explain = "PLAN="
         + "EnumerableInterpreter\n"
         + "  BindableTableScan(table=[[s, beatles]], filters=[[=($0, 4)]], projects=[[2, 1]]";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles", table)))
+        .with(newSchema("s", PairList.of("beatles", table)))
         .query("select \"k\",\"j\" from \"s\".\"beatles\" where \"i\" = 4")
         .explainContains(explain)
         .returnsUnordered("k=1940; j=John",
@@ -193,8 +197,7 @@ public class ScannableTableTest {
 
   /** A filter on a {@link org.apache.calcite.schema.ProjectableFilterableTable}
    * with two columns, and a project in the query (NonCooperative). */
-  @Test void testProjectableFilterableWithProjectFilterNonCooperative()
-      throws Exception {
+  @Test void testProjectableFilterableWithProjectFilterNonCooperative() {
     final StringBuilder buf = new StringBuilder();
     final Table table = new BeatlesProjectableFilterableTable(buf, false);
     final String explain = "PLAN="
@@ -202,7 +205,7 @@ public class ScannableTableTest {
         + "  BindableTableScan(table=[[s, beatles]], filters=[[>($2, 1941)]], "
         + "projects=[[0, 2]])";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles", table)))
+        .with(newSchema("s", PairList.of("beatles", table)))
         .query("select \"i\",\"k\" from \"s\".\"beatles\" where \"k\" > 1941")
         .explainContains(explain)
         .returnsUnordered("i=4; k=1942",
@@ -215,13 +218,13 @@ public class ScannableTableTest {
    * {@link org.apache.calcite.schema.ProjectableFilterableTable}. The table
    * refuses to execute the filter, so Calcite should add a pull up and
    * transform the filter (projecting the column needed by the filter). */
-  @Test void testPFTableRefusesFilterCooperative() throws Exception {
+  @Test void testPFTableRefusesFilterCooperative() {
     final StringBuilder buf = new StringBuilder();
     final Table table = new BeatlesProjectableFilterableTable(buf, false);
     final String explain = "PLAN=EnumerableInterpreter\n"
         + "  BindableTableScan(table=[[s, beatles2]], filters=[[=($0, 4)]], projects=[[2]])";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles2", table)))
+        .with(newSchema("s", PairList.of("beatles2", table)))
         .query("select \"k\" from \"s\".\"beatles2\" where \"i\" = 4")
         .explainContains(explain)
         .returnsUnordered("k=1940",
@@ -237,7 +240,7 @@ public class ScannableTableTest {
         + "  EnumerableInterpreter\n"
         + "    BindableTableScan(table=[[s, beatles]], filters=[[>($0, 1)]], projects=[[2]])";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles", table)))
+        .with(newSchema("s", PairList.of("beatles", table)))
         .query("select max(\"k\") as m from \"s\".\"beatles\" where \"i\" > 1")
         .explainContains(explain)
         .returnsUnordered("M=1943");
@@ -256,7 +259,7 @@ public class ScannableTableTest {
         + "    BindableTableScan(table=[[s, beatles]], filters=[[>($2, 1900)]], "
         + "projects=[[0]])";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles", table)))
+        .with(newSchema("s", PairList.of("beatles", table)))
         .query(sql)
         .explainContains(explain)
         .returnsUnordered("i=4; C=2",
@@ -278,13 +281,14 @@ public class ScannableTableTest {
         + "    EnumerableInterpreter\n"
         + "      BindableTableScan(table=[[s, beatles]], filters=[[=($2, 1940)]], projects=[[2, 0]])";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles", table)))
+        .with(newSchema("s", PairList.of("beatles", table)))
         .query(sql)
         .explainContains(explain)
         .returnsUnordered("k=1940; C=2");
   }
 
-  private static Pair<Integer, Object> getFilter(boolean cooperative, List<RexNode> filters) {
+  private static @Nullable Pair<Integer, Object> getFilter(boolean cooperative,
+      List<RexNode> filters) {
     final Iterator<RexNode> filterIter = filters.iterator();
     while (filterIter.hasNext()) {
       final RexNode node = filterIter.next();
@@ -312,14 +316,14 @@ public class ScannableTableTest {
    * <a href="https://issues.apache.org/jira/browse/CALCITE-458">[CALCITE-458]
    * ArrayIndexOutOfBoundsException when using just a single column in
    * interpreter</a>. */
-  @Test void testPFTableRefusesFilterSingleColumn() throws Exception {
+  @Test void testPFTableRefusesFilterSingleColumn() {
     final StringBuilder buf = new StringBuilder();
     final Table table = new BeatlesProjectableFilterableTable(buf, false);
     final String explain = "PLAN="
         + "EnumerableInterpreter\n"
         + "  BindableTableScan(table=[[s, beatles2]], filters=[[>($2, 1941)]], projects=[[2]])";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles2", table)))
+        .with(newSchema("s", PairList.of("beatles2", table)))
         .query("select \"k\" from \"s\".\"beatles2\" where \"k\" > 1941")
         .explainContains(explain)
         .returnsUnordered("k=1942",
@@ -330,7 +334,7 @@ public class ScannableTableTest {
   /** Test case for
    * <a href="https://issues.apache.org/jira/browse/CALCITE-3405">[CALCITE-3405]
    * Prune columns for ProjectableFilterable when project is not simple mapping</a>. */
-  @Test void testPushNonSimpleMappingProject() throws Exception {
+  @Test void testPushNonSimpleMappingProject() {
     final StringBuilder buf = new StringBuilder();
     final Table table = new BeatlesProjectableFilterableTable(buf, true);
     final String explain = "PLAN="
@@ -339,7 +343,7 @@ public class ScannableTableTest {
         + "  EnumerableInterpreter\n"
         + "    BindableTableScan(table=[[s, beatles]], projects=[[2, 0]])";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles", table)))
+        .with(newSchema("s", PairList.of("beatles", table)))
         .query("select \"k\", \"i\", \"k\", \"i\"+\"i\" \"ii\", 3 from \"s\".\"beatles\"")
         .explainContains(explain)
         .returnsUnordered(
@@ -353,7 +357,7 @@ public class ScannableTableTest {
   /** Test case for
    * <a href="https://issues.apache.org/jira/browse/CALCITE-3405">[CALCITE-3405]
    * Prune columns for ProjectableFilterable when project is not simple mapping</a>. */
-  @Test void testPushSimpleMappingProject() throws Exception {
+  @Test void testPushSimpleMappingProject() {
     final StringBuilder buf = new StringBuilder();
     final Table table = new BeatlesProjectableFilterableTable(buf, true);
     // Note that no redundant Project on EnumerableInterpreter
@@ -361,7 +365,7 @@ public class ScannableTableTest {
         + "EnumerableInterpreter\n"
         + "  BindableTableScan(table=[[s, beatles]], projects=[[2, 0]])";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles", table)))
+        .with(newSchema("s", PairList.of("beatles", table)))
         .query("select \"k\", \"i\" from \"s\".\"beatles\"")
         .explainContains(explain)
         .returnsUnordered(
@@ -377,7 +381,7 @@ public class ScannableTableTest {
    * Stack overflow error thrown when running join query</a>
    * Test two ProjectableFilterableTable can join and produce right plan.
    */
-  @Test void testProjectableFilterableTableJoin() throws Exception {
+  @Test void testProjectableFilterableTableJoin() {
     final StringBuilder buf = new StringBuilder();
     final String explain = "PLAN="
         + "EnumerableNestedLoopJoin(condition=[true], joinType=[inner])\n"
@@ -388,8 +392,12 @@ public class ScannableTableTest {
     CalciteAssert.that()
             .with(
               newSchema("s",
-                  Pair.of("b1", new BeatlesProjectableFilterableTable(buf, true)),
-                  Pair.of("b2", new BeatlesProjectableFilterableTable(buf, true))))
+                  PairList.<String, Table>builder()
+                      .add("b1",
+                          new BeatlesProjectableFilterableTable(buf, true))
+                      .add("b2",
+                          new BeatlesProjectableFilterableTable(buf, true))
+                      .build()))
             .query("select * from \"s\".\"b1\", \"s\".\"b2\" "
                     + "where \"s\".\"b1\".\"i\" = 10 and \"s\".\"b2\".\"i\" = 10 "
                     + "and \"s\".\"b1\".\"i\" = \"s\".\"b2\".\"i\"")
@@ -399,7 +407,7 @@ public class ScannableTableTest {
   /** Test case for
    * <a href="https://issues.apache.org/jira/browse/CALCITE-5019">[CALCITE-5019]
    * Avoid multiple scans when table is ProjectableFilterableTable</a>.*/
-  @Test void testProjectableFilterableWithScanCounter() throws Exception {
+  @Test void testProjectableFilterableWithScanCounter() {
     final StringBuilder buf = new StringBuilder();
     final BeatlesProjectableFilterableTable table =
         new BeatlesProjectableFilterableTable(buf, false);
@@ -407,7 +415,7 @@ public class ScannableTableTest {
         + "EnumerableInterpreter\n"
         + "  BindableTableScan(table=[[s, beatles]], filters=[[=($0, 4)]], projects=[[1]]";
     CalciteAssert.that()
-        .with(newSchema("s", Pair.of("beatles", table)))
+        .with(newSchema("s", PairList.of("beatles", table)))
         .query("select \"j\" from \"s\".\"beatles\" where \"i\" = 4")
         .explainContains(explain)
         .returnsUnordered("j=John", "j=Paul");
@@ -482,7 +490,7 @@ public class ScannableTableTest {
       final String explain = "PLAN="
           + "BindableTableScan(table=[[s, beatles]], filters=[[=($1, 'John')]], projects=[[1]])";
       CalciteAssert.that()
-          .with(newSchema("s", Pair.of("beatles", table)))
+          .with(newSchema("s", PairList.of("beatles", table)))
           .query("select \"j\" from \"s\".\"beatles\" where \"j\" = 'John'")
           .explainContains(explain)
           .returnsUnordered("j=John");
@@ -492,14 +500,12 @@ public class ScannableTableTest {
   }
 
   protected ConnectionPostProcessor newSchema(final String schemaName,
-      Pair<String, Table>... tables) {
+      PairList<String, Table> tables) {
     return connection -> {
       CalciteConnection con = connection.unwrap(CalciteConnection.class);
       SchemaPlus rootSchema = con.getRootSchema();
       SchemaPlus schema = rootSchema.add(schemaName, new AbstractSchema());
-      for (Pair<String, Table> t : tables) {
-        schema.add(t.left, t.right);
-      }
+      tables.forEach(schema::add);
       connection.setSchema(schemaName);
       return connection;
     };
@@ -531,6 +537,7 @@ public class ScannableTableTest {
       return typeFactory.builder()
           .add("i", SqlTypeName.INTEGER)
           .add("j", SqlTypeName.VARCHAR)
+          .add("k", SqlTypeName.INTEGER)
           .build();
     }
 
@@ -614,10 +621,10 @@ public class ScannableTableTest {
   private static Enumerator<Object[]> tens() {
     return new Enumerator<Object[]>() {
       int row = -1;
-      Object[] current;
+      Object @Nullable[] current;
 
       public Object[] current() {
-        return current;
+        return requireNonNull(current, "current");
       }
 
       public boolean moveNext() {
@@ -676,14 +683,15 @@ public class ScannableTableTest {
   };
 
   private static Enumerator<Object[]> beatles(final StringBuilder buf,
-      final Pair<Integer, Object> filter, final int[] projects) {
+      @Nullable final Pair<Integer, Object> filter,
+      final int @Nullable[] projects) {
     return new Enumerator<Object[]>() {
       int row = -1;
       int returnCount = 0;
-      Object[] current;
+      Object @Nullable[] current;
 
       public Object[] current() {
-        return current;
+        return requireNonNull(current, "current");
       }
 
       public boolean moveNext() {


[calcite] 01/02: In Puffin, allow an action to test whether it is looking at the last line of a source

Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jhyde pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git

commit 357f06bb855047c6d5ac8229291033b905eb4a68
Author: Julian Hyde <jh...@apache.org>
AuthorDate: Sat Aug 19 22:42:53 2023 -0700

    In Puffin, allow an action to test whether it is looking at the last line of a source
---
 .../main/java/org/apache/calcite/util/Puffin.java  | 46 +++++++++++++++++-----
 .../java/org/apache/calcite/test/PuffinTest.java   | 13 ++++--
 2 files changed, 47 insertions(+), 12 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/util/Puffin.java b/core/src/main/java/org/apache/calcite/util/Puffin.java
index f0a42a9c56..8b5d6e286c 100644
--- a/core/src/main/java/org/apache/calcite/util/Puffin.java
+++ b/core/src/main/java/org/apache/calcite/util/Puffin.java
@@ -144,16 +144,38 @@ public class Puffin {
    * @param <F> Type of state that is created when we start processing a file
    */
   public interface Line<G, F> {
+    /** Returns the global state. */
     G globalState();
+
+    /** Returns the state of the current file. */
     F state();
+
+    /** Returns the ordinal of the current line in its source. */
     int fnr();
+
+    /** Returns the filename of the current source. */
     String filename();
+
+    /** Returns the current source. */
     Source source();
+
+    /** Returns whether the current line starts with {@code prefix}. */
     boolean startsWith(String prefix);
-    boolean contains(CharSequence s);
+
+    /** Returns whether the current line starts with {@code seek}. */
+    boolean contains(CharSequence seek);
+
+    /** Returns whether the current line ends with {@code suffix}. */
     boolean endsWith(String suffix);
+
+    /** Returns whether the current line matches {@code regex}. */
     boolean matches(String regex);
+
+    /** Returns the text of the current line. */
     String line();
+
+    /** Returns whether the current line is the last line in its source. */
+    boolean isLast();
   }
 
   /** Context for executing a Puffin program within a given file.
@@ -176,6 +198,9 @@ public class Puffin {
      * number of records". */
     int fnr = 0;
 
+    /** Holds whether the current line is the last line in its source. */
+    boolean last;
+
     Context(PrintWriter out, Source source,
         Function<String, Pattern> patternCache, G globalState,
         F fileState) {
@@ -227,6 +252,10 @@ public class Puffin {
       return source().toString();
     }
 
+    @Override public boolean isLast() {
+      return last;
+    }
+
     @Override public Source source() {
       return source;
     }
@@ -293,7 +322,7 @@ public class Puffin {
       final Source source0 = Sources.of("");
       final F fileState0 = fileStateFactory.apply(globalState);
       final ContextImpl<G, F> x0 =
-          new ContextImpl<G, F>(out, source0, patternCache, globalState,
+          new ContextImpl<>(out, source0, patternCache, globalState,
               fileState0);
       beforeList.forEach(action -> action.accept(x0));
       sources.forEach(source -> execute(globalState, source, out));
@@ -306,16 +335,15 @@ public class Puffin {
            BufferedReader br = new BufferedReader(r)) {
         final F fileState = fileStateFactory.apply(globalState);
         final ContextImpl<G, F> x =
-            new ContextImpl<G, F>(out, source, patternCache, globalState,
+            new ContextImpl<>(out, source, patternCache, globalState,
                 fileState);
         beforeSourceList.forEach(action -> action.accept(x));
-        for (;;) {
-          String lineText = br.readLine();
-          if (lineText == null) {
-            break;
-          }
+        String nextLine = br.readLine();
+        while (nextLine != null) {
+          x.line = nextLine;
           ++x.fnr;
-          x.line = lineText;
+          nextLine = br.readLine();
+          x.last = nextLine == null;
           onLineList.forEach((predicate, action) -> {
             if (predicate.test(x)) {
               action.accept(x);
diff --git a/core/src/test/java/org/apache/calcite/test/PuffinTest.java b/core/src/test/java/org/apache/calcite/test/PuffinTest.java
index 3cbebca6a8..69a4712c54 100644
--- a/core/src/test/java/org/apache/calcite/test/PuffinTest.java
+++ b/core/src/test/java/org/apache/calcite/test/PuffinTest.java
@@ -71,6 +71,10 @@ public class PuffinTest {
         Puffin.builder(GlobalState::new, u -> new AtomicInteger())
             .add(line -> true,
                 line -> line.state().incrementAndGet())
+            .add(Puffin.Line::isLast,
+                line ->
+                    line.globalState().messages.add("last line of "
+                        + line.filename() + " is [" + line.line() + "]"))
             .beforeSource(context -> {
               final GlobalState g = context.globalState();
               g.beforeSourceCount.incrementAndGet();
@@ -99,16 +103,19 @@ public class PuffinTest {
         program.execute(
             Stream.of(Sources.of("a\nb\n"),
                 Sources.of("a\n"),
-                Sources.of("a\nb\nc\n")),
+                Sources.of("a\nb\nc\n\n")),
             new PrintWriter(sw));
-    assertThat(g.messages, hasSize(7));
-    assertThat(g.messages, hasItem("3 lines"));
+    assertThat(g.messages, hasSize(10));
+    assertThat(g.messages, hasItem("4 lines"));
     assertThat(g.messages, hasItem("2 lines"));
     assertThat(g.messages, hasItem("1 lines"));
     assertThat(g.messages, hasItem("3 after sources"));
     assertThat(g.messages, hasItem("3 before sources"));
     assertThat(g.messages, hasItem("1 before"));
     assertThat(g.messages, hasItem("1 after"));
+    assertThat(g.messages, hasItem("last line of GuavaCharSource{memory} is [b]"));
+    assertThat(g.messages, hasItem("last line of GuavaCharSource{memory} is [a]"));
+    assertThat(g.messages, hasItem("last line of GuavaCharSource{memory} is []"));
     assertThat(sw, hasToString(""));
   }