You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2022/05/02 13:35:28 UTC
[iotdb] branch master updated: [IOTDB-2993][Bugfix] Support IN, LIKE, and REGEXP expressions in Transform Operator (#5766)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 38ca619b69 [IOTDB-2993][Bugfix] Support IN, LIKE, and REGEXP expressions in Transform Operator (#5766)
38ca619b69 is described below
commit 38ca619b69b071ab05f79b123e4c58f19c6fe14b
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Mon May 2 21:35:23 2022 +0800
[IOTDB-2993][Bugfix] Support IN, LIKE, and REGEXP expressions in Transform Operator (#5766)
---
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 3 +-
.../iotdb/db/integration/IoTDBNestedQueryIT.java | 6 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 49 ++++++-----
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 49 ++++++-----
.../db/query/expression/unary/InExpression.java | 11 ++-
.../udf/core/transformer/unary/InTransformer.java | 98 ++++++++++++++++++++--
6 files changed, 162 insertions(+), 54 deletions(-)
diff --git a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index 6a3abb9c73..92125573ae 100644
--- a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -841,7 +841,7 @@ expression
| leftExpression=expression (PLUS | MINUS) rightExpression=expression
| leftExpression=expression (OPERATOR_GT | OPERATOR_GTE | OPERATOR_LT | OPERATOR_LTE | OPERATOR_DEQ | OPERATOR_NEQ) rightExpression=expression
| unaryBeforeRegularOrLikeExpression=expression (REGEXP | LIKE) STRING_LITERAL
- | unaryBeforeInExpression=expression OPERATOR_IN LR_BRACKET constant (COMMA constant)* RR_BRACKET
+ | unaryBeforeInExpression=expression OPERATOR_NOT? OPERATOR_IN LR_BRACKET constant (COMMA constant)* RR_BRACKET
| leftExpression=expression OPERATOR_AND rightExpression=expression
| leftExpression=expression OPERATOR_OR rightExpression=expression
;
@@ -919,7 +919,6 @@ resultColumn
;
-
// From Clause
fromClause
diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBNestedQueryIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBNestedQueryIT.java
index dbf69fbe81..c74ce71b6d 100644
--- a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBNestedQueryIT.java
+++ b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBNestedQueryIT.java
@@ -34,7 +34,6 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -587,20 +586,19 @@ public class IoTDBNestedQueryIT {
}
@Test
- @Ignore
public void testRegularLikeInExpressions() {
try (Connection connection =
DriverManager.getConnection(
Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
Statement statement = connection.createStatement()) {
String query =
- "SELECT ((CAST(s1, 'type'='TEXT') LIKE '_') REGEXP '*') IN ('4', '2', '3') "
+ "SELECT ((CAST(s1, 'type'='TEXT') LIKE '_') REGEXP '[0-9]') IN ('4', '2', '3') "
+ "FROM root.vehicle.d1";
try (ResultSet rs = statement.executeQuery(query)) {
for (int i = 2; i <= 4; i++) {
Assert.assertTrue(rs.next());
Assert.assertEquals(i, rs.getLong(1));
- Assert.assertEquals(i, rs.getLong(2));
+ Assert.assertEquals(String.valueOf(i), rs.getString(2));
}
Assert.assertFalse(rs.next());
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
index e77b1a8a1f..6326c67883 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
@@ -99,6 +99,8 @@ import org.apache.iotdb.db.query.expression.binary.SubtractionExpression;
import org.apache.iotdb.db.query.expression.leaf.ConstantOperand;
import org.apache.iotdb.db.query.expression.leaf.TimeSeriesOperand;
import org.apache.iotdb.db.query.expression.multi.FunctionExpression;
+import org.apache.iotdb.db.query.expression.unary.InExpression;
+import org.apache.iotdb.db.query.expression.unary.LikeExpression;
import org.apache.iotdb.db.query.expression.unary.LogicNotExpression;
import org.apache.iotdb.db.query.expression.unary.NegationExpression;
import org.apache.iotdb.db.query.expression.unary.RegularExpression;
@@ -115,6 +117,7 @@ import java.util.ArrayList;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -1711,23 +1714,6 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
return parseExpression(context.unaryInBracket, inWithoutNull);
}
- if (context.constant() != null && !context.constant().isEmpty()) {
- return parseConstantOperand(context.constant(0));
- }
-
- if (context.time != null) {
- throw new UnsupportedOperationException();
- }
-
- if (context.fullPathInExpression() != null) {
- return new TimeSeriesOperand(
- parseFullPathInExpression(context.fullPathInExpression(), inWithoutNull));
- }
-
- if (context.functionName() != null) {
- return parseFunctionExpression(context, inWithoutNull);
- }
-
if (context.expressionAfterUnaryOperator != null) {
if (context.MINUS() != null) {
return new NegationExpression(
@@ -1799,6 +1785,23 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
return parseInExpression(context, inWithoutNull);
}
+ if (context.functionName() != null) {
+ return parseFunctionExpression(context, inWithoutNull);
+ }
+
+ if (context.fullPathInExpression() != null) {
+ return new TimeSeriesOperand(
+ parseFullPathInExpression(context.fullPathInExpression(), inWithoutNull));
+ }
+
+ if (context.time != null) {
+ throw new UnsupportedOperationException();
+ }
+
+ if (context.constant() != null && !context.constant().isEmpty()) {
+ return parseConstantOperand(context.constant(0));
+ }
+
throw new UnsupportedOperationException();
}
@@ -1843,11 +1846,19 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
}
private Expression parseLikeExpression(ExpressionContext context, boolean inWithoutNull) {
- throw new UnsupportedOperationException();
+ return new LikeExpression(
+ parseExpression(context.unaryBeforeRegularOrLikeExpression, inWithoutNull),
+ parseStringLiteral(context.STRING_LITERAL().getText()));
}
private Expression parseInExpression(ExpressionContext context, boolean inWithoutNull) {
- throw new UnsupportedOperationException();
+ Expression childExpression = parseExpression(context.unaryBeforeInExpression, inWithoutNull);
+ LinkedHashSet<String> values = new LinkedHashSet<>();
+ for (ConstantContext constantContext : context.constant()) {
+ String text = constantContext.getText();
+ values.add(constantContext.STRING_LITERAL() != null ? parseStringLiteral(text) : text);
+ }
+ return new InExpression(childExpression, context.OPERATOR_NOT() != null, values);
}
private Expression parseConstantOperand(ConstantContext constantContext) {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java b/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
index cb45bf2385..e58f1f06d0 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
@@ -145,6 +145,8 @@ import org.apache.iotdb.db.query.expression.binary.SubtractionExpression;
import org.apache.iotdb.db.query.expression.leaf.ConstantOperand;
import org.apache.iotdb.db.query.expression.leaf.TimeSeriesOperand;
import org.apache.iotdb.db.query.expression.multi.FunctionExpression;
+import org.apache.iotdb.db.query.expression.unary.InExpression;
+import org.apache.iotdb.db.query.expression.unary.LikeExpression;
import org.apache.iotdb.db.query.expression.unary.LogicNotExpression;
import org.apache.iotdb.db.query.expression.unary.NegationExpression;
import org.apache.iotdb.db.query.expression.unary.RegularExpression;
@@ -166,6 +168,7 @@ import java.util.Arrays;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -2547,23 +2550,6 @@ public class IoTDBSqlVisitor extends IoTDBSqlParserBaseVisitor<Operator> {
return parseExpression(context.unaryInBracket, inWithoutNull);
}
- if (context.constant() != null && !context.constant().isEmpty()) {
- return parseConstantOperand(context.constant(0));
- }
-
- if (context.time != null) {
- throw new UnsupportedOperationException();
- }
-
- if (context.fullPathInExpression() != null) {
- return new TimeSeriesOperand(
- parseFullPathInExpression(context.fullPathInExpression(), inWithoutNull));
- }
-
- if (context.functionName() != null) {
- return parseFunctionExpression(context, inWithoutNull);
- }
-
if (context.expressionAfterUnaryOperator != null) {
if (context.MINUS() != null) {
return new NegationExpression(
@@ -2635,6 +2621,23 @@ public class IoTDBSqlVisitor extends IoTDBSqlParserBaseVisitor<Operator> {
return parseInExpression(context, inWithoutNull);
}
+ if (context.functionName() != null) {
+ return parseFunctionExpression(context, inWithoutNull);
+ }
+
+ if (context.fullPathInExpression() != null) {
+ return new TimeSeriesOperand(
+ parseFullPathInExpression(context.fullPathInExpression(), inWithoutNull));
+ }
+
+ if (context.time != null) {
+ throw new UnsupportedOperationException();
+ }
+
+ if (context.constant() != null && !context.constant().isEmpty()) {
+ return parseConstantOperand(context.constant(0));
+ }
+
throw new UnsupportedOperationException();
}
@@ -2679,11 +2682,19 @@ public class IoTDBSqlVisitor extends IoTDBSqlParserBaseVisitor<Operator> {
}
private Expression parseLikeExpression(ExpressionContext context, boolean inWithoutNull) {
- throw new UnsupportedOperationException();
+ return new LikeExpression(
+ parseExpression(context.unaryBeforeRegularOrLikeExpression, inWithoutNull),
+ parseStringLiteral(context.STRING_LITERAL().getText()));
}
private Expression parseInExpression(ExpressionContext context, boolean inWithoutNull) {
- throw new UnsupportedOperationException();
+ Expression childExpression = parseExpression(context.unaryBeforeInExpression, inWithoutNull);
+ LinkedHashSet<String> values = new LinkedHashSet<>();
+ for (ConstantContext constantContext : context.constant()) {
+ String text = constantContext.getText();
+ values.add(constantContext.STRING_LITERAL() != null ? parseStringLiteral(text) : text);
+ }
+ return new InExpression(childExpression, context.OPERATOR_NOT() != null, values);
}
private Expression parseConstantOperand(ConstantContext constantContext) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/InExpression.java b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/InExpression.java
index a5a5b97111..2d311cba1d 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/InExpression.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/InExpression.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.db.query.expression.Expression;
import org.apache.iotdb.db.query.expression.ExpressionType;
import org.apache.iotdb.db.query.udf.core.reader.LayerPointReader;
import org.apache.iotdb.db.query.udf.core.transformer.Transformer;
+import org.apache.iotdb.db.query.udf.core.transformer.unary.InTransformer;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.nio.ByteBuffer;
@@ -31,15 +32,18 @@ import java.util.LinkedHashSet;
public class InExpression extends UnaryExpression {
+ private final boolean isNotIn;
private final LinkedHashSet<String> values;
- protected InExpression(Expression expression, LinkedHashSet<String> values) {
+ public InExpression(Expression expression, boolean isNotIn, LinkedHashSet<String> values) {
super(expression);
+ this.isNotIn = isNotIn;
this.values = values;
}
public InExpression(ByteBuffer byteBuffer) {
super(Expression.deserialize(byteBuffer));
+ isNotIn = ReadWriteIOUtils.readBool(byteBuffer);
final int size = ReadWriteIOUtils.readInt(byteBuffer);
values = new LinkedHashSet<>();
for (int i = 0; i < size; ++i) {
@@ -67,17 +71,18 @@ public class InExpression extends UnaryExpression {
@Override
protected Transformer constructTransformer(LayerPointReader pointReader) {
- throw new UnsupportedOperationException();
+ return new InTransformer(pointReader, isNotIn, values);
}
@Override
protected Expression constructExpression(Expression childExpression) {
- return new InExpression(childExpression, values);
+ return new InExpression(childExpression, isNotIn, values);
}
@Override
protected void serialize(ByteBuffer byteBuffer) {
super.serialize(byteBuffer);
+ ReadWriteIOUtils.write(isNotIn, byteBuffer);
ReadWriteIOUtils.write(values.size(), byteBuffer);
for (String value : values) {
ReadWriteIOUtils.write(value, byteBuffer);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/udf/core/transformer/unary/InTransformer.java b/server/src/main/java/org/apache/iotdb/db/query/udf/core/transformer/unary/InTransformer.java
index 46d59c0701..74dab92467 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/udf/core/transformer/unary/InTransformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/udf/core/transformer/unary/InTransformer.java
@@ -32,6 +32,8 @@ public class InTransformer extends UnaryTransformer {
private final TSDataType dataType;
+ private final Satisfy satisfy;
+
private Set<Integer> intSet;
private Set<Long> longSet;
private Set<Float> floatSet;
@@ -39,9 +41,10 @@ public class InTransformer extends UnaryTransformer {
private Set<Boolean> booleanSet;
private Set<String> stringSet;
- public InTransformer(LayerPointReader layerPointReader, Set<String> values) {
+ public InTransformer(LayerPointReader layerPointReader, boolean isNotIn, Set<String> values) {
super(layerPointReader);
dataType = layerPointReader.getDataType();
+ satisfy = isNotIn ? new NotInSatisfy() : new InSatisfy();
initTypedSet(values);
}
@@ -96,7 +99,7 @@ public class InTransformer extends UnaryTransformer {
switch (dataType) {
case INT32:
int intValue = layerPointReader.currentInt();
- if (intSet.contains(intValue)) {
+ if (satisfy.of(intValue)) {
cachedInt = intValue;
} else {
currentNull = true;
@@ -104,7 +107,7 @@ public class InTransformer extends UnaryTransformer {
break;
case INT64:
long longValue = layerPointReader.currentLong();
- if (longSet.contains(longValue)) {
+ if (satisfy.of(longValue)) {
cachedLong = longValue;
} else {
currentNull = true;
@@ -112,7 +115,7 @@ public class InTransformer extends UnaryTransformer {
break;
case FLOAT:
float floatValue = layerPointReader.currentFloat();
- if (floatSet.contains(floatValue)) {
+ if (satisfy.of(floatValue)) {
cachedFloat = floatValue;
} else {
currentNull = true;
@@ -120,7 +123,7 @@ public class InTransformer extends UnaryTransformer {
break;
case DOUBLE:
double doubleValue = layerPointReader.currentDouble();
- if (doubleSet.contains(doubleValue)) {
+ if (satisfy.of(doubleValue)) {
cachedDouble = doubleValue;
} else {
currentNull = true;
@@ -128,7 +131,7 @@ public class InTransformer extends UnaryTransformer {
break;
case BOOLEAN:
boolean booleanValue = layerPointReader.currentBoolean();
- if (booleanSet.contains(booleanValue)) {
+ if (satisfy.of(booleanValue)) {
cachedBoolean = booleanValue;
} else {
currentNull = true;
@@ -136,7 +139,7 @@ public class InTransformer extends UnaryTransformer {
break;
case TEXT:
Binary binaryValue = layerPointReader.currentBinary();
- if (stringSet.contains(binaryValue.getStringValue())) {
+ if (satisfy.of(binaryValue.getStringValue())) {
cachedBinary = binaryValue;
} else {
currentNull = true;
@@ -146,4 +149,85 @@ public class InTransformer extends UnaryTransformer {
throw new QueryProcessException("unsupported data type: " + layerPointReader.getDataType());
}
}
+
+ private interface Satisfy {
+
+ boolean of(int intValue);
+
+ boolean of(long longValue);
+
+ boolean of(float floatValue);
+
+ boolean of(double doubleValue);
+
+ boolean of(boolean booleanValue);
+
+ boolean of(String stringValue);
+ }
+
+ private class InSatisfy implements Satisfy {
+
+ @Override
+ public boolean of(int intValue) {
+ return intSet.contains(intValue);
+ }
+
+ @Override
+ public boolean of(long longValue) {
+ return longSet.contains(longValue);
+ }
+
+ @Override
+ public boolean of(float floatValue) {
+ return floatSet.contains(floatValue);
+ }
+
+ @Override
+ public boolean of(double doubleValue) {
+ return doubleSet.contains(doubleValue);
+ }
+
+ @Override
+ public boolean of(boolean booleanValue) {
+ return booleanSet.contains(booleanValue);
+ }
+
+ @Override
+ public boolean of(String stringValue) {
+ return stringSet.contains(stringValue);
+ }
+ }
+
+ private class NotInSatisfy implements Satisfy {
+
+ @Override
+ public boolean of(int intValue) {
+ return !intSet.contains(intValue);
+ }
+
+ @Override
+ public boolean of(long longValue) {
+ return !longSet.contains(longValue);
+ }
+
+ @Override
+ public boolean of(float floatValue) {
+ return !floatSet.contains(floatValue);
+ }
+
+ @Override
+ public boolean of(double doubleValue) {
+ return !doubleSet.contains(doubleValue);
+ }
+
+ @Override
+ public boolean of(boolean booleanValue) {
+ return !booleanSet.contains(booleanValue);
+ }
+
+ @Override
+ public boolean of(String stringValue) {
+ return !stringSet.contains(stringValue);
+ }
+ }
}