You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2021/06/10 09:26:07 UTC
[calcite] 01/01: [CALCITE-4606] In Elasticsearch adapter,
translate SEARCH RexCall to termsQuery (Jacky Yin)
This is an automated email from the ASF dual-hosted git repository.
jhyde pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 8213277dc3e4e1ea0ddb64f7df9b34bf73305e3d
Author: jacky <ja...@eoitek.com>
AuthorDate: Thu May 27 18:29:01 2021 +0800
[CALCITE-4606] In Elasticsearch adapter, translate SEARCH RexCall to termsQuery (Jacky Yin)
Translate SEARCH RexCall(In/NotIn) to termsQuery in ES.
Close apache/calcite#2420
---
.../adapter/elasticsearch/PredicateAnalyzer.java | 112 ++++++++++++++++++++-
.../adapter/elasticsearch/AggregationTest.java | 43 +++++++-
2 files changed, 153 insertions(+), 2 deletions(-)
diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java
index 3eea1c4..b8e5c45 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/PredicateAnalyzer.java
@@ -29,24 +29,31 @@ import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlSyntax;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.Sarg;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
+import com.google.common.collect.Range;
+import java.util.ArrayList;
import java.util.GregorianCalendar;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import static org.apache.calcite.adapter.elasticsearch.QueryBuilders.boolQuery;
import static org.apache.calcite.adapter.elasticsearch.QueryBuilders.existsQuery;
import static org.apache.calcite.adapter.elasticsearch.QueryBuilders.rangeQuery;
import static org.apache.calcite.adapter.elasticsearch.QueryBuilders.regexpQuery;
import static org.apache.calcite.adapter.elasticsearch.QueryBuilders.termQuery;
+import static org.apache.calcite.adapter.elasticsearch.QueryBuilders.termsQuery;
import static java.lang.String.format;
+import static java.util.Objects.requireNonNull;
/**
* Query predicate analyzer. Uses visitor pattern to traverse existing expression
@@ -179,6 +186,13 @@ class PredicateAnalyzer {
default:
return false;
}
+ case INTERNAL:
+ switch (call.getKind()) {
+ case SEARCH:
+ return canBeTranslatedToTermsQuery(call);
+ default:
+ return false;
+ }
case FUNCTION_ID:
case FUNCTION_STAR:
default:
@@ -186,6 +200,36 @@ class PredicateAnalyzer {
}
}
+ /**
+ * There are three types of the Sarg included in SEARCH RexCall:
+ * 1) Sarg is points (In ('a', 'b', 'c' ...)).
+ * In this case the search call can be translated to terms Query
+ * 2) Sarg is complementedPoints (Not in ('a', 'b')).
+ * In this case the search call can be translated to MustNot terms Query
+ * 3) Sarg is real Range( > 1 and <= 10).
+ * In this case the search call should be translated to rang Query
+ * Currently only the 1) and 2) cases are supported.
+ * @param search SEARCH RexCall
+ * @return true if it isSearchWithPoints or isSearchWithComplementedPoints, other false
+ */
+ static boolean canBeTranslatedToTermsQuery(RexCall search) {
+ return isSearchWithPoints(search) || isSearchWithComplementedPoints(search);
+ }
+
+ @SuppressWarnings("BetaApi")
+ static boolean isSearchWithPoints(RexCall search) {
+ RexLiteral literal = (RexLiteral) search.getOperands().get(1);
+ final Sarg<?> sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg");
+ return sarg.isPoints();
+ }
+
+ @SuppressWarnings("BetaApi")
+ static boolean isSearchWithComplementedPoints(RexCall search) {
+ RexLiteral literal = (RexLiteral) search.getOperands().get(1);
+ final Sarg<?> sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg");
+ return sarg.isComplementedPoints();
+ }
+
@Override public Expression visitCall(RexCall call) {
SqlSyntax syntax = call.getOperator().getSyntax();
@@ -201,6 +245,8 @@ class PredicateAnalyzer {
return postfix(call);
case PREFIX:
return prefix(call);
+ case INTERNAL:
+ return binary(call);
case SPECIAL:
switch (call.getKind()) {
case CAST:
@@ -350,6 +396,12 @@ class PredicateAnalyzer {
return QueryExpression.create(pair.getKey()).gte(pair.getValue());
}
return QueryExpression.create(pair.getKey()).lte(pair.getValue());
+ case SEARCH:
+ if (isSearchWithComplementedPoints(call)) {
+ return QueryExpression.create(pair.getKey()).notIn(pair.getValue());
+ } else {
+ return QueryExpression.create(pair.getKey()).in(pair.getValue());
+ }
default:
break;
}
@@ -533,6 +585,10 @@ class PredicateAnalyzer {
public abstract QueryExpression equals(LiteralExpression literal);
+ public abstract QueryExpression in(LiteralExpression literal);
+
+ public abstract QueryExpression notIn(LiteralExpression literal);
+
public abstract QueryExpression notEquals(LiteralExpression literal);
public abstract QueryExpression gt(LiteralExpression literal);
@@ -677,6 +733,14 @@ class PredicateAnalyzer {
@Override public QueryExpression isTrue() {
throw new PredicateAnalyzerException("isTrue cannot be applied to a compound expression");
}
+
+ @Override public QueryExpression in(LiteralExpression literal) {
+ throw new PredicateAnalyzerException("in cannot be applied to a compound expression");
+ }
+
+ @Override public QueryExpression notIn(LiteralExpression literal) {
+ throw new PredicateAnalyzerException("notIn cannot be applied to a compound expression");
+ }
}
/**
@@ -797,6 +861,18 @@ class PredicateAnalyzer {
builder = termQuery(getFieldReference(), true);
return this;
}
+
+ @Override public QueryExpression in(LiteralExpression literal) {
+ Iterable<?> iterable = (Iterable<?>) literal.value();
+ builder = termsQuery(getFieldReference(), iterable);
+ return this;
+ }
+
+ @Override public QueryExpression notIn(LiteralExpression literal) {
+ Iterable<?> iterable = (Iterable<?>) literal.value();
+ builder = boolQuery().mustNot(termsQuery(getFieldReference(), iterable));
+ return this;
+ }
}
@@ -896,7 +972,9 @@ class PredicateAnalyzer {
Object value() {
- if (isIntegral()) {
+ if (isSarg()) {
+ return sargValue();
+ } else if (isIntegral()) {
return longValue();
} else if (isFloatingPoint()) {
return doubleValue();
@@ -925,6 +1003,10 @@ class PredicateAnalyzer {
return SqlTypeName.CHAR_TYPES.contains(literal.getType().getSqlTypeName());
}
+ public boolean isSarg() {
+ return SqlTypeName.SARG.getName().equalsIgnoreCase(literal.getTypeName().getName());
+ }
+
long longValue() {
return ((Number) literal.getValue()).longValue();
}
@@ -941,6 +1023,34 @@ class PredicateAnalyzer {
return RexLiteral.stringValue(literal);
}
+ @SuppressWarnings("BetaApi")
+ List<Object> sargValue() {
+ final Sarg sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg");
+ final RelDataType type = literal.getType();
+ List<Object> values = new ArrayList<>();
+ final SqlTypeName sqlTypeName = type.getSqlTypeName();
+ if (sarg.isPoints()) {
+ Set<Range> ranges = sarg.rangeSet.asRanges();
+ ranges.forEach(range ->
+ values.add(sargPointValue(range.lowerEndpoint(), sqlTypeName)));
+ } else if (sarg.isComplementedPoints()) {
+ Set<Range> ranges = sarg.negate().rangeSet.asRanges();
+ ranges.forEach(range ->
+ values.add(sargPointValue(range.lowerEndpoint(), sqlTypeName)));
+ }
+ return values;
+ }
+
+ Object sargPointValue(Object point, SqlTypeName sqlTypeName) {
+ switch (sqlTypeName) {
+ case CHAR:
+ case VARCHAR:
+ return ((NlsString) point).getValue();
+ default:
+ return point;
+ }
+ }
+
Object rawValue() {
return literal.getValue();
}
diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationTest.java
index f79edc8..385ad26 100644
--- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationTest.java
+++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/AggregationTest.java
@@ -47,7 +47,6 @@ import java.util.Map;
/**
* Testing Elasticsearch aggregation transformations.
*/
-@Disabled("RestClient often timeout in PR CI")
@ResourceLock(value = "elasticsearch-scrolls", mode = ResourceAccessMode.READ)
class AggregationTest {
@@ -112,6 +111,33 @@ class AggregationTest {
};
}
+ /**
+ * Currently the patterns like below will be converted to Search in range
+ * which is not supported in elastic search adapter.
+ * (val1 >= 10 and val1 <= 20)
+ * (val1 <= 10 or val1 >=20)
+ * (val1 <= 10) or (val1 > 15 and val1 <= 20)
+ * So disable this test case until the translation from Search in range
+ * to rang Query in ES is implemented.
+ */
+ @Disabled
+ @Test void searchInRange() {
+ CalciteAssert.that()
+ .with(newConnectionFactory())
+ .query("select count(*) from view where val1 >= 10 and val1 <=20")
+ .returns("EXPR$0=1\n");
+
+ CalciteAssert.that()
+ .with(newConnectionFactory())
+ .query("select count(*) from view where val1 <= 10 or val1 >=20")
+ .returns("EXPR$0=2\n");
+
+ CalciteAssert.that()
+ .with(newConnectionFactory())
+ .query("select count(*) from view where val1 <= 10 or (val1 > 15 and val1 <= 20)")
+ .returns("EXPR$0=2\n");
+ }
+
@Test void countStar() {
CalciteAssert.that()
.with(newConnectionFactory())
@@ -130,6 +156,21 @@ class AggregationTest {
.with(newConnectionFactory())
.query("select count(*) from view where cat1 in ('a', 'b')")
.returns("EXPR$0=2\n");
+
+ CalciteAssert.that()
+ .with(newConnectionFactory())
+ .query("select count(*) from view where val1 in (10, 20)")
+ .returns("EXPR$0=0\n");
+
+ CalciteAssert.that()
+ .with(newConnectionFactory())
+ .query("select count(*) from view where cat4 in ('2018-01-01', '2019-12-12')")
+ .returns("EXPR$0=2\n");
+
+ CalciteAssert.that()
+ .with(newConnectionFactory())
+ .query("select count(*) from view where cat4 not in ('2018-01-01', '2019-12-12')")
+ .returns("EXPR$0=1\n");
}
@Test void all() {