You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by as...@apache.org on 2024/06/08 08:02:18 UTC
(calcite) branch main updated: [CALCITE-6429] Arrow adapter should default to the Enumerable convention for unsupported filters
This is an automated email from the ASF dual-hosted git repository.
asolimando pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push:
new 10b3f53270 [CALCITE-6429] Arrow adapter should default to the Enumerable convention for unsupported filters
10b3f53270 is described below
commit 10b3f532704fa526b32f96e9ef9fd4e11c698704
Author: Alessandro Solimando <al...@gmail.com>
AuthorDate: Tue Mar 12 17:41:06 2024 +0100
[CALCITE-6429] Arrow adapter should default to the Enumerable convention for unsupported filters
---
.../apache/calcite/adapter/arrow/ArrowRules.java | 9 ++-
.../calcite/adapter/arrow/ArrowTranslator.java | 8 +--
.../calcite/adapter/arrow/ArrowAdapterTest.java | 79 ++++++++++++++++------
.../src/main/java/org/apache/calcite/util/Bug.java | 20 ++++++
4 files changed, 88 insertions(+), 28 deletions(-)
diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java
index 42e47e8be2..b70e709648 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowRules.java
@@ -84,8 +84,13 @@ public class ArrowRules {
final Filter filter = call.rel(0);
if (filter.getTraitSet().contains(Convention.NONE)) {
- final RelNode converted = convert(filter);
- call.transformTo(converted);
+ try {
+ final RelNode converted = convert(filter);
+ call.transformTo(converted);
+ } catch (UnsupportedOperationException e) {
+ // skip rule application when hitting an unsupported feature,
+ // allowing a plan in the Enumerable convention to be generated
+ }
}
}
diff --git a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java
index 5b68a130da..900e470571 100644
--- a/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java
+++ b/arrow/src/main/java/org/apache/calcite/adapter/arrow/ArrowTranslator.java
@@ -67,7 +67,7 @@ class ArrowTranslator {
if (disjunctions.size() == 1) {
return translateAnd(disjunctions.get(0));
} else {
- throw new AssertionError("cannot translate " + condition);
+ throw new UnsupportedOperationException("Unsupported disjunctive condition " + condition);
}
}
@@ -128,7 +128,7 @@ class ArrowTranslator {
case GREATER_THAN_OR_EQUAL:
return translateBinary("greater_than_or_equal_to", "<=", (RexCall) node);
default:
- throw new AssertionError("cannot translate " + node);
+ throw new UnsupportedOperationException("Unsupported binary operator " + node);
}
}
@@ -147,7 +147,7 @@ class ArrowTranslator {
if (expression != null) {
return expression;
}
- throw new AssertionError("cannot translate op " + op + " call " + call);
+ throw new UnsupportedOperationException("Unsupported binary operator " + call);
}
/** Translates a call to a binary operator. Returns null on failure. */
@@ -197,6 +197,6 @@ class ArrowTranslator {
} else if (String.class.equals(literal.getClass())) {
return "string";
}
- throw new AssertionError("Invalid literal");
+ throw new UnsupportedOperationException("Unsupported literal " + literal);
}
}
diff --git a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
index 19e7bdfd5d..6dd8b75cd4 100644
--- a/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
+++ b/arrow/src/test/java/org/apache/calcite/adapter/arrow/ArrowAdapterTest.java
@@ -22,6 +22,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeSystem;
import org.apache.calcite.schema.Table;
import org.apache.calcite.test.CalciteAssert;
+import org.apache.calcite.util.Bug;
import org.apache.calcite.util.Sources;
import com.google.common.collect.ImmutableList;
@@ -238,15 +239,24 @@ class ArrowAdapterTest {
.explainContains(plan);
}
- @Disabled("OR is not supported yet")
@Test void testArrowProjectFieldsWithDisjunctiveFilter() {
String sql = "select \"intField\", \"stringField\"\n"
+ "from arrowdata\n"
+ "where \"intField\"=12 or \"stringField\"='12'";
- String plan = "PLAN=ArrowToEnumerableConverter\n"
- + " ArrowProject(intField=[$0], stringField=[$1])\n"
- + " ArrowFilter(condition=[OR(=($0, 12), =($1, '12'))])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+ String plan;
+ if (Bug.CALCITE_6293_FIXED) {
+ plan = "PLAN=ArrowToEnumerableConverter\n"
+ + " ArrowProject(intField=[$0], stringField=[$1])\n"
+ + " ArrowFilter(condition=[OR(=($0, 12), =($1, '12'))])\n"
+ + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+ } else {
+ plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[12], "
+ + "expr#3=[=($t0, $t2)], expr#4=['12':VARCHAR], expr#5=[=($t1, $t4)], "
+ + "expr#6=[OR($t3, $t5)], proj#0..1=[{exprs}], $condition=[$t6])\n"
+ + " ArrowToEnumerableConverter\n"
+ + " ArrowProject(intField=[$0], stringField=[$1])\n"
+ + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+ }
String result = "intField=12; stringField=12\n";
CalciteAssert.that()
@@ -256,15 +266,23 @@ class ArrowAdapterTest {
.explainContains(plan);
}
- @Disabled("IN is not supported as OR is not supported yet")
@Test void testArrowProjectFieldsWithInFilter() {
String sql = "select \"intField\", \"stringField\"\n"
+ "from arrowdata\n"
+ "where \"intField\" in (0, 1, 2)";
- String plan = "PLAN=ArrowToEnumerableConverter\n"
- + " ArrowProject(intField=[$0], stringField=[$1])\n"
- + " ArrowFilter(condition=[OR(=($0, 0), =($0, 1), =($0, 2))])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+ String plan;
+ if (Bug.CALCITE_6294_FIXED) {
+ plan = "PLAN=ArrowToEnumerableConverter\n"
+ + " ArrowProject(intField=[$0], stringField=[$1])\n"
+ + " ArrowFilter(condition=[OR(=($0, 0), =($0, 1), =($0, 2))])\n"
+ + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+ } else {
+ plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[Sarg[0, 1, 2]], "
+ + "expr#3=[SEARCH($t0, $t2)], proj#0..1=[{exprs}], $condition=[$t3])\n"
+ + " ArrowToEnumerableConverter\n"
+ + " ArrowProject(intField=[$0], stringField=[$1])\n"
+ + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+ }
String result = "intField=0; stringField=0\n"
+ "intField=1; stringField=1\n"
+ "intField=2; stringField=2\n";
@@ -276,19 +294,28 @@ class ArrowAdapterTest {
.explainContains(plan);
}
- @Disabled("IS NOT NULL is not supported yet")
@Test void testArrowProjectFieldsWithIsNotNullFilter() {
String sql = "select \"intField\", \"stringField\"\n"
+ "from arrowdata\n"
+ "where \"intField\" is not null\n"
+ "order by \"intField\"\n"
+ "limit 1";
- String plan = "PLAN=EnumerableLimit(fetch=[1])\n"
- + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"
- + " ArrowToEnumerableConverter\n"
- + " ArrowProject(intField=[$0], stringField=[$1])\n"
- + " ArrowFilter(condition=[IS NOT NULL($0)])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+ String plan;
+ if (Bug.CALCITE_6295_FIXED) {
+ plan = "PLAN=EnumerableLimit(fetch=[1])\n"
+ + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"
+ + " ArrowToEnumerableConverter\n"
+ + " ArrowProject(intField=[$0], stringField=[$1])\n"
+ + " ArrowFilter(condition=[IS NOT NULL($0)])\n"
+ + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+ } else {
+ plan = "PLAN=EnumerableCalc(expr#0..3=[{inputs}], proj#0..1=[{exprs}])\n"
+ + " EnumerableLimit(fetch=[1])\n"
+ + " EnumerableSort(sort0=[$0], dir0=[ASC])\n"
+ + " EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NOT NULL($t0)], proj#0..3=[{exprs}], $condition=[$t4])\n"
+ + " ArrowToEnumerableConverter\n"
+ + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+ }
String result = "intField=0; stringField=0\n";
CalciteAssert.that()
@@ -298,15 +325,23 @@ class ArrowAdapterTest {
.explainContains(plan);
}
- @Disabled("IS NULL is not supported yet")
@Test void testArrowProjectFieldsWithIsNullFilter() {
String sql = "select \"intField\", \"stringField\"\n"
+ "from arrowdata\n"
+ "where \"intField\" is null";
- String plan = "ArrowToEnumerableConverter\n"
- + " ArrowProject(intField=[$0], stringField=[$1])\n"
- + " ArrowFilter(condition=[IS NOT NULL($0)])\n"
- + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+ String plan;
+ if (Bug.CALCITE_6296_FIXED) {
+ plan = "ArrowToEnumerableConverter\n"
+ + " ArrowProject(intField=[$0], stringField=[$1])\n"
+ + " ArrowFilter(condition=[IS NOT NULL($0)])\n"
+ + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+ } else {
+ plan = "PLAN=EnumerableCalc(expr#0..1=[{inputs}],"
+ + " expr#2=[IS NULL($t0)], proj#0..1=[{exprs}], $condition=[$t2])\n"
+ + " ArrowToEnumerableConverter\n"
+ + " ArrowProject(intField=[$0], stringField=[$1])\n"
+ + " ArrowTableScan(table=[[ARROW, ARROWDATA]], fields=[[0, 1, 2, 3]])\n\n";
+ }
CalciteAssert.that()
.with(arrow)
diff --git a/core/src/main/java/org/apache/calcite/util/Bug.java b/core/src/main/java/org/apache/calcite/util/Bug.java
index 331922dce5..3e5385ee02 100644
--- a/core/src/main/java/org/apache/calcite/util/Bug.java
+++ b/core/src/main/java/org/apache/calcite/util/Bug.java
@@ -221,6 +221,26 @@ public abstract class Bug {
* is fixed. */
public static final boolean CALCITE_6391_FIXED = false;
+ /** Whether
+ * <a href="https://issues.apache.org/jira/browse/CALCITE/issues/CALCITE-6293">
+ * [CALCITE-6293] Support OR condition in Arrow adapter</a> is fixed. */
+ public static final boolean CALCITE_6293_FIXED = false;
+
+ /** Whether
+ * <a href="https://issues.apache.org/jira/browse/CALCITE/issues/CALCITE-6294">
+ * [CALCITE-6294] Support IN filter in Arrow adapter</a> is fixed. */
+ public static final boolean CALCITE_6294_FIXED = false;
+
+ /** Whether
+ * <a href="https://issues.apache.org/jira/browse/CALCITE/issues/CALCITE-6295">
+ * [CALCITE-6295] Support IS NOT NULL in Arrow adapter</a> is fixed. */
+ public static final boolean CALCITE_6295_FIXED = false;
+
+ /** Whether
+ * <a href="https://issues.apache.org/jira/browse/CALCITE/issues/CALCITE-6296">
+ * [CALCITE-6296] Support IS NULL in Arrow adapter</a> is fixed. */
+ public static final boolean CALCITE_6296_FIXED = false;
+
/**
* Use this to flag temporary code.
*/