You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/03/14 09:08:03 UTC
[iotdb] branch master updated: [IOTDB-5643] Add REPLACE as a built-in scalar function
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2b48e40706 [IOTDB-5643] Add REPLACE as a built-in scalar function
2b48e40706 is described below
commit 2b48e407065d8005bd347f62ccef71a5cd408db6
Author: Liao Lanyu <14...@qq.com>
AuthorDate: Tue Mar 14 17:07:55 2023 +0800
[IOTDB-5643] Add REPLACE as a built-in scalar function
---
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 8 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
.../itbase/constant/BuiltinScalarFunctionEnum.java | 1 +
.../scalar/IoTDBReplaceFunctionIT.java | 171 +++++++++++++++++++++
.../commons/udf/builtin/BuiltinScalarFunction.java | 5 +
.../org/apache/iotdb/db/constant/SqlConstant.java | 3 +
.../plan/expression/multi/FunctionExpression.java | 61 ++++----
.../multi/builtin/BuiltInScalarFunctionHelper.java | 23 ++-
.../BuiltInScalarFunctionHelperFactory.java | 11 +-
.../{CastHelper.java => CastFunctionHelper.java} | 20 +--
.../{DiffHelper.java => DiffFunctionHelper.java} | 11 +-
...{DiffHelper.java => ReplaceFunctionHelper.java} | 56 ++++---
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 28 +++-
.../scalar/CastFunctionColumnTransformer.java | 20 +--
.../scalar/ReplaceFunctionColumnTransformer.java | 57 +++++++
.../unary/scalar/CastFunctionTransformer.java | 20 +--
.../unary/scalar/ReplaceFunctionTransformer.java | 50 ++++++
18 files changed, 450 insertions(+), 100 deletions(-)
diff --git a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4
index 06eb8d3d71..3ebcd8f92d 100644
--- a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4
+++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4
@@ -153,6 +153,7 @@ keyWords
| RENAME
| RESAMPLE
| RESOURCE
+ | REPLACE
| REVOKE
| ROLE
| RUNNING
diff --git a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index 57b6b8e2cb..064d1151fd 100644
--- a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -960,7 +960,7 @@ expression
| constant
| time=(TIME | TIMESTAMP)
| fullPathInExpression
- | CAST LR_BRACKET castInput=expression AS attributeValue RR_BRACKET
+ | scalarFunctionExpression
| functionName LR_BRACKET expression (COMMA expression)* RR_BRACKET
| (PLUS | MINUS | OPERATOR_NOT) expressionAfterUnaryOperator=expression
| leftExpression=expression (STAR | DIV | MOD) rightExpression=expression
@@ -979,6 +979,12 @@ functionName
| COUNT
;
+scalarFunctionExpression
+ : CAST LR_BRACKET castInput=expression AS attributeValue RR_BRACKET
+ | REPLACE LR_BRACKET text=expression COMMA from=STRING_LITERAL COMMA to=STRING_LITERAL RR_BRACKET
+ ;
+
+
containsExpression
: name=attributeKey OPERATOR_CONTAINS value=attributeValue
;
diff --git a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
index f349831619..3791743780 100644
--- a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
+++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
@@ -554,6 +554,10 @@ RESOURCE
: R E S O U R C E
;
+REPLACE
+ : R E P L A C E
+ ;
+
REVOKE
: R E V O K E
;
diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/constant/BuiltinScalarFunctionEnum.java b/integration-test/src/main/java/org/apache/iotdb/itbase/constant/BuiltinScalarFunctionEnum.java
index bd38e182cd..9518a2d546 100644
--- a/integration-test/src/main/java/org/apache/iotdb/itbase/constant/BuiltinScalarFunctionEnum.java
+++ b/integration-test/src/main/java/org/apache/iotdb/itbase/constant/BuiltinScalarFunctionEnum.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.itbase.constant;
public enum BuiltinScalarFunctionEnum {
DIFF("diff"),
CAST("cast"),
+ REPLACE("replace"),
;
private final String functionName;
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBReplaceFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBReplaceFunctionIT.java
new file mode 100644
index 0000000000..8afb9b4389
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/builtinfunction/scalar/IoTDBReplaceFunctionIT.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it.builtinfunction.scalar;
+
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import static org.apache.iotdb.db.it.utils.TestUtils.prepareData;
+import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest;
+import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR;
+import static org.junit.Assert.fail;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({LocalStandaloneIT.class, ClusterIT.class})
+public class IoTDBReplaceFunctionIT {
+ private static final String[] SQLs =
+ new String[] {
+ "CREATE DATABASE root.sg",
+ "CREATE TIMESERIES root.sg.s1 WITH DATATYPE=TEXT, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.sg.s2 WITH DATATYPE=INT32, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.sg.s2 WITH DATATYPE=INT64, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.sg.s2 WITH DATATYPE=FLOAT, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.sg.s2 WITH DATATYPE=DOUBLE, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.sg.s2 WITH DATATYPE=BOOLEAN, ENCODING=PLAIN",
+ "INSERT INTO root.sg(timestamp,s1,s2,s3,s4,s5,s6) values(1, 'abcd', 1, 1, 1, 1, true)",
+ "INSERT INTO root.sg(timestamp,s1) values(2, 'test\\\\')",
+ "INSERT INTO root.sg(timestamp,s1) values(3, 'abcd\\\\')",
+ "flush"
+ };
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvFactory.getEnv().initClusterEnvironment();
+ prepareData(SQLs);
+ registerUDF();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ private static void registerUDF() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "create function constvalue as 'org.apache.iotdb.db.query.udf.example.ConstValue'");
+ } catch (SQLException throwable) {
+ fail(throwable.getMessage());
+ }
+ }
+
+ @Test
+ public void testNewTransformer() {
+ String[] expectedHeader =
+ new String[] {
+ TIMESTAMP_STR, "REPLACE(root.sg.s1, 'ab', 'AB')", "REPLACE(root.sg.s1, '\\', 'a')"
+ };
+ String[] retArray =
+ new String[] {
+ "1,ABcd,abcd,", "2,test\\\\,testaa,", "3,ABcd\\\\,abcdaa,",
+ };
+ resultSetEqualTest(
+ "select REPLACE(s1, 'ab', 'AB'), REPLACE(s1, '\\', 'a') from root.sg",
+ expectedHeader,
+ retArray);
+ }
+
+ @Test
+ public void testOldTransformer() {
+ String[] expectedHeader =
+ new String[] {
+ TIMESTAMP_STR,
+ "constvalue(root.sg.s1)",
+ "REPLACE(root.sg.s1, 'ab', 'AB')",
+ "REPLACE(root.sg.s1, '\\', 'a')"
+ };
+ String[] retArray =
+ new String[] {
+ "1,1,ABcd,abcd,", "2,1,test\\\\,testaa,", "3,1,ABcd\\\\,abcdaa,",
+ };
+ resultSetEqualTest(
+ "select constvalue(s1),REPLACE(s1, 'ab', 'AB'), REPLACE(s1, '\\', 'a') from root.sg",
+ expectedHeader,
+ retArray);
+ }
+
+ @Test
+ public void testWithoutFromOrTo() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ try {
+ statement.execute("select REPLACE(s1, 'b') from root.sg");
+ fail();
+ } catch (Exception ignored) {
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
+
+ @Test
+ public void testWrongInputType() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ try {
+ statement.execute("select REPLACE(s2, 'a', 'b') from root.sg");
+ fail();
+ } catch (Exception ignored) {
+ }
+
+ try {
+ statement.execute("select REPLACE(s3, 'a', 'b') from root.sg");
+ fail();
+ } catch (Exception ignored) {
+ }
+
+ try {
+ statement.execute("select REPLACE(s4, 'a', 'b') from root.sg");
+ fail();
+ } catch (Exception ignored) {
+ }
+
+ try {
+ statement.execute("select REPLACE(s5, 'a', 'b') from root.sg");
+ fail();
+ } catch (Exception ignored) {
+ }
+
+ try {
+ statement.execute("select REPLACE(s6, 'a', 'b') from root.sg");
+ fail();
+ } catch (Exception ignored) {
+ }
+
+ } catch (SQLException e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
+}
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/BuiltinScalarFunction.java b/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/BuiltinScalarFunction.java
index 03cd7abe1b..691f225eb5 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/BuiltinScalarFunction.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/BuiltinScalarFunction.java
@@ -29,6 +29,7 @@ import java.util.stream.Collectors;
public enum BuiltinScalarFunction {
DIFF("diff"),
CAST("cast"),
+ REPLACE("replace"),
;
private final String functionName;
@@ -47,6 +48,10 @@ public enum BuiltinScalarFunction {
.map(BuiltinScalarFunction::getFunctionName)
.collect(Collectors.toList()));
+ public static boolean contains(String functionName) {
+ return NATIVE_FUNCTION_NAMES.contains(functionName.toLowerCase());
+ }
+
/**
* We shouldn't apply these functions to each DataRegion for one device, because these functions
* need all data to calculate correct result. So we need to collect all data for one device in one
diff --git a/server/src/main/java/org/apache/iotdb/db/constant/SqlConstant.java b/server/src/main/java/org/apache/iotdb/db/constant/SqlConstant.java
index 3f9236b0f1..020073bb12 100644
--- a/server/src/main/java/org/apache/iotdb/db/constant/SqlConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/constant/SqlConstant.java
@@ -63,6 +63,9 @@ public class SqlConstant {
public static final String CAST_FUNCTION = "CAST";
public static final String CAST_TYPE = "type";
+ public static final String REPLACE_FUNCTION = "REPLACE";
+ public static final String REPLACE_FROM = "FROM";
+ public static final String REPLACE_TO = "TO";
public static String[] getSingleRootArray() {
return SINGLE_ROOT_ARRAY;
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/FunctionExpression.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/FunctionExpression.java
index 1ca65fb373..720a17e80e 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/FunctionExpression.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/FunctionExpression.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.db.mpp.common.NodeRef;
import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.expression.ExpressionType;
import org.apache.iotdb.db.mpp.plan.expression.leaf.TimeSeriesOperand;
+import org.apache.iotdb.db.mpp.plan.expression.multi.builtin.BuiltInScalarFunctionHelperFactory;
import org.apache.iotdb.db.mpp.plan.expression.visitor.ExpressionVisitor;
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.InputLocation;
import org.apache.iotdb.db.mpp.transformation.dag.memory.LayerMemoryAssigner;
@@ -48,8 +49,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.stream.Collectors;
-import static org.apache.iotdb.db.constant.SqlConstant.CAST_FUNCTION;
-
public class FunctionExpression extends Expression {
private FunctionType functionType;
@@ -264,35 +263,12 @@ public class FunctionExpression extends Expression {
}
}
if (!functionAttributes.isEmpty()) {
- // currently only the header of cast function is different
- // this if-else branch can be extracted into a method if more new functions have different
- // header
- if (functionName.equalsIgnoreCase(CAST_FUNCTION)) {
- // Cast has only one attribute
- builder.append(" AS ");
- builder.append(functionAttributes.entrySet().iterator().next().getValue());
+ // Some builtin-scalar function may have different header.
+ if (BuiltinScalarFunction.contains(functionName)) {
+ BuiltInScalarFunctionHelperFactory.createHelper(functionName)
+ .appendFunctionAttributes(!expressions.isEmpty(), builder, functionAttributes);
} else {
- if (!expressions.isEmpty()) {
- builder.append(", ");
- }
- Iterator<Entry<String, String>> iterator = functionAttributes.entrySet().iterator();
- Entry<String, String> entry = iterator.next();
- builder
- .append("\"")
- .append(entry.getKey())
- .append("\"=\"")
- .append(entry.getValue())
- .append("\"");
- while (iterator.hasNext()) {
- entry = iterator.next();
- builder
- .append(", ")
- .append("\"")
- .append(entry.getKey())
- .append("\"=\"")
- .append(entry.getValue())
- .append("\"");
- }
+ appendAttributes(!expressions.isEmpty(), builder, functionAttributes);
}
}
parametersString = builder.toString();
@@ -300,6 +276,31 @@ public class FunctionExpression extends Expression {
return parametersString;
}
+ public static void appendAttributes(
+ boolean hasExpression, StringBuilder builder, Map<String, String> functionAttributes) {
+ if (hasExpression) {
+ builder.append(", ");
+ }
+ Iterator<Entry<String, String>> iterator = functionAttributes.entrySet().iterator();
+ Entry<String, String> entry = iterator.next();
+ builder
+ .append("\"")
+ .append(entry.getKey())
+ .append("\"=\"")
+ .append(entry.getValue())
+ .append("\"");
+ while (iterator.hasNext()) {
+ entry = iterator.next();
+ builder
+ .append(", ")
+ .append("\"")
+ .append(entry.getKey())
+ .append("\"=\"")
+ .append(entry.getValue())
+ .append("\"");
+ }
+ }
+
@Override
public ExpressionType getExpressionType() {
return ExpressionType.FUNCTION;
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/BuiltInScalarFunctionHelper.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/BuiltInScalarFunctionHelper.java
index 8966610e41..7525a4f155 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/BuiltInScalarFunctionHelper.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/BuiltInScalarFunctionHelper.java
@@ -26,6 +26,10 @@ import org.apache.iotdb.db.mpp.transformation.dag.column.ColumnTransformer;
import org.apache.iotdb.db.mpp.transformation.dag.transformer.Transformer;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import java.util.Map;
+
+import static org.apache.iotdb.db.mpp.plan.parser.ASTVisitor.checkFunctionExpressionInputSize;
+
/**
* This interface defines the methods that FunctionExpression may use if it is a FunctionExpression
* representing a built-in function
@@ -35,8 +39,11 @@ public interface BuiltInScalarFunctionHelper extends BuiltInFunctionHelper {
* Check if the input size is correct. For example, function DIFF only supports one column as
* input. Throw {@link SemanticException} if the input size is not correct.
*/
- void checkBuiltInScalarFunctionInputSize(FunctionExpression functionExpression)
- throws SemanticException;
+ default void checkBuiltInScalarFunctionInputSize(FunctionExpression functionExpression)
+ throws SemanticException {
+ checkFunctionExpressionInputSize(
+ functionExpression.getExpressionString(), functionExpression.getExpressions().size(), 1);
+ }
/**
* Check if the input TsDataType is correct. Throw {@link SemanticException} if the type is not
@@ -71,4 +78,16 @@ public interface BuiltInScalarFunctionHelper extends BuiltInFunctionHelper {
*/
Transformer getBuiltInScalarFunctionTransformer(
FunctionExpression expression, LayerPointReader layerPointReader);
+
+ /**
+ * Some builtin-scalar function may have a different header. This method will be called by {@link
+ * FunctionExpression#getExpressionStringInternal()} )}
+ *
+ * @param builder String builder in FunctionExpression. Append function attributes through it.
+ * @param functionAttributes attributes of the function
+ */
+ default void appendFunctionAttributes(
+ boolean hasExpression, StringBuilder builder, Map<String, String> functionAttributes) {
+ FunctionExpression.appendAttributes(hasExpression, builder, functionAttributes);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/BuiltInScalarFunctionHelperFactory.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/BuiltInScalarFunctionHelperFactory.java
index 1454e5cca4..f7d09a81df 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/BuiltInScalarFunctionHelperFactory.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/BuiltInScalarFunctionHelperFactory.java
@@ -19,17 +19,20 @@
package org.apache.iotdb.db.mpp.plan.expression.multi.builtin;
-import org.apache.iotdb.db.mpp.plan.expression.multi.builtin.helper.CastHelper;
-import org.apache.iotdb.db.mpp.plan.expression.multi.builtin.helper.DiffHelper;
+import org.apache.iotdb.db.mpp.plan.expression.multi.builtin.helper.CastFunctionHelper;
+import org.apache.iotdb.db.mpp.plan.expression.multi.builtin.helper.DiffFunctionHelper;
+import org.apache.iotdb.db.mpp.plan.expression.multi.builtin.helper.ReplaceFunctionHelper;
public class BuiltInScalarFunctionHelperFactory {
public static BuiltInScalarFunctionHelper createHelper(String functionName) {
functionName = functionName.toUpperCase();
switch (functionName) {
case "DIFF":
- return new DiffHelper();
+ return new DiffFunctionHelper();
case "CAST":
- return new CastHelper();
+ return new CastFunctionHelper();
+ case "REPLACE":
+ return new ReplaceFunctionHelper();
default:
throw new IllegalArgumentException(
String.format("Invalid scalar function [%s].", functionName));
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/CastHelper.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/CastFunctionHelper.java
similarity index 92%
rename from server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/CastHelper.java
rename to server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/CastFunctionHelper.java
index 39eb31ba5f..ff6a5e85db 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/CastHelper.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/CastFunctionHelper.java
@@ -30,17 +30,11 @@ import org.apache.iotdb.db.mpp.transformation.dag.transformer.unary.scalar.CastF
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.type.TypeFactory;
-import static org.apache.iotdb.db.constant.SqlConstant.CAST_TYPE;
-import static org.apache.iotdb.db.mpp.plan.parser.ASTVisitor.checkFunctionExpressionInputSize;
+import java.util.Map;
-public class CastHelper implements BuiltInScalarFunctionHelper {
- @Override
- public void checkBuiltInScalarFunctionInputSize(FunctionExpression functionExpression)
- throws SemanticException {
- checkFunctionExpressionInputSize(
- functionExpression.getExpressionString(), functionExpression.getExpressions().size(), 1);
- }
+import static org.apache.iotdb.db.constant.SqlConstant.CAST_TYPE;
+public class CastFunctionHelper implements BuiltInScalarFunctionHelper {
@Override
public void checkBuiltInScalarFunctionInputDataType(TSDataType tsDataType)
throws SemanticException {}
@@ -69,6 +63,14 @@ public class CastHelper implements BuiltInScalarFunctionHelper {
layerPointReader, this.getBuiltInScalarFunctionReturnType(expression));
}
+ @Override
+ public void appendFunctionAttributes(
+ boolean hasExpression, StringBuilder builder, Map<String, String> functionAttributes) {
+ // Cast has only one attribute
+ builder.append(" AS ");
+ builder.append(functionAttributes.entrySet().iterator().next().getValue());
+ }
+
public static int castLongToInt(long value) {
if (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) {
throw new RuntimeException(
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/DiffHelper.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/DiffFunctionHelper.java
similarity index 86%
copy from server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/DiffHelper.java
copy to server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/DiffFunctionHelper.java
index 16c88a4205..a705323e7c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/DiffHelper.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/DiffFunctionHelper.java
@@ -30,16 +30,7 @@ import org.apache.iotdb.db.mpp.transformation.dag.transformer.unary.scalar.DiffF
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.type.TypeFactory;
-import static org.apache.iotdb.db.mpp.plan.parser.ASTVisitor.checkFunctionExpressionInputSize;
-
-public class DiffHelper implements BuiltInScalarFunctionHelper {
-
- @Override
- public void checkBuiltInScalarFunctionInputSize(FunctionExpression functionExpression)
- throws SemanticException {
- checkFunctionExpressionInputSize(
- functionExpression.getExpressionString(), functionExpression.getExpressions().size(), 1);
- }
+public class DiffFunctionHelper implements BuiltInScalarFunctionHelper {
@Override
public void checkBuiltInScalarFunctionInputDataType(TSDataType tsDataType)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/DiffHelper.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/ReplaceFunctionHelper.java
similarity index 58%
rename from server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/DiffHelper.java
rename to server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/ReplaceFunctionHelper.java
index 16c88a4205..e7242bc20d 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/DiffHelper.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/builtin/helper/ReplaceFunctionHelper.java
@@ -24,54 +24,68 @@ import org.apache.iotdb.db.mpp.plan.expression.multi.FunctionExpression;
import org.apache.iotdb.db.mpp.plan.expression.multi.builtin.BuiltInScalarFunctionHelper;
import org.apache.iotdb.db.mpp.transformation.api.LayerPointReader;
import org.apache.iotdb.db.mpp.transformation.dag.column.ColumnTransformer;
-import org.apache.iotdb.db.mpp.transformation.dag.column.unary.scalar.DiffFunctionColumnTransformer;
+import org.apache.iotdb.db.mpp.transformation.dag.column.unary.scalar.ReplaceFunctionColumnTransformer;
import org.apache.iotdb.db.mpp.transformation.dag.transformer.Transformer;
-import org.apache.iotdb.db.mpp.transformation.dag.transformer.unary.scalar.DiffFunctionTransformer;
+import org.apache.iotdb.db.mpp.transformation.dag.transformer.unary.scalar.ReplaceFunctionTransformer;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.type.TypeFactory;
-import static org.apache.iotdb.db.mpp.plan.parser.ASTVisitor.checkFunctionExpressionInputSize;
+import java.util.Map;
-public class DiffHelper implements BuiltInScalarFunctionHelper {
-
- @Override
- public void checkBuiltInScalarFunctionInputSize(FunctionExpression functionExpression)
- throws SemanticException {
- checkFunctionExpressionInputSize(
- functionExpression.getExpressionString(), functionExpression.getExpressions().size(), 1);
- }
+import static org.apache.iotdb.db.constant.SqlConstant.REPLACE_FROM;
+import static org.apache.iotdb.db.constant.SqlConstant.REPLACE_TO;
+public class ReplaceFunctionHelper implements BuiltInScalarFunctionHelper {
@Override
public void checkBuiltInScalarFunctionInputDataType(TSDataType tsDataType)
throws SemanticException {
- if (tsDataType.isNumeric()) {
+ if (tsDataType.equals(TSDataType.TEXT)) {
return;
}
throw new SemanticException(
- "Input series of Scalar function [DIFF] only supports numeric data types [INT32, INT64, FLOAT, DOUBLE]");
+ "Input series of Scalar function [REPLACE] only support text data type.");
}
@Override
public TSDataType getBuiltInScalarFunctionReturnType(FunctionExpression functionExpression) {
- return TSDataType.DOUBLE;
+ return TSDataType.TEXT;
}
@Override
public ColumnTransformer getBuiltInScalarFunctionColumnTransformer(
FunctionExpression expression, ColumnTransformer columnTransformer) {
- return new DiffFunctionColumnTransformer(
- TypeFactory.getType(TSDataType.DOUBLE),
+ checkFromAndToAttributes(expression);
+ return new ReplaceFunctionColumnTransformer(
+ TypeFactory.getType(TSDataType.TEXT),
columnTransformer,
- Boolean.parseBoolean(
- expression.getFunctionAttributes().getOrDefault("ignoreNull", "true")));
+ expression.getFunctionAttributes().get(REPLACE_FROM),
+ expression.getFunctionAttributes().get(REPLACE_TO));
}
@Override
public Transformer getBuiltInScalarFunctionTransformer(
FunctionExpression expression, LayerPointReader layerPointReader) {
- return new DiffFunctionTransformer(
+ checkFromAndToAttributes(expression);
+ return new ReplaceFunctionTransformer(
layerPointReader,
- Boolean.parseBoolean(
- expression.getFunctionAttributes().getOrDefault("ignoreNull", "true")));
+ expression.getFunctionAttributes().get(REPLACE_FROM),
+ expression.getFunctionAttributes().get(REPLACE_TO));
+ }
+
+ @Override
+ public void appendFunctionAttributes(
+ boolean hasExpression, StringBuilder builder, Map<String, String> functionAttributes) {
+ builder.append(", '");
+ builder.append(functionAttributes.get(REPLACE_FROM));
+ builder.append("', '");
+ builder.append(functionAttributes.get(REPLACE_TO));
+ builder.append("'");
+ }
+
+ private void checkFromAndToAttributes(FunctionExpression expression) {
+ if (!expression.getFunctionAttributes().containsKey(REPLACE_FROM)
+ || !expression.getFunctionAttributes().containsKey(REPLACE_TO)) {
+ throw new SemanticException("Function REPLACE must specify from and to component.");
+ }
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
index 8cddc33597..811b377e7d 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
@@ -208,6 +208,9 @@ import java.util.stream.Collectors;
import static org.apache.iotdb.db.constant.SqlConstant.CAST_FUNCTION;
import static org.apache.iotdb.db.constant.SqlConstant.CAST_TYPE;
+import static org.apache.iotdb.db.constant.SqlConstant.REPLACE_FROM;
+import static org.apache.iotdb.db.constant.SqlConstant.REPLACE_FUNCTION;
+import static org.apache.iotdb.db.constant.SqlConstant.REPLACE_TO;
import static org.apache.iotdb.db.metadata.MetadataConstant.ALL_RESULT_NODES;
/** Parse AST to Statement. */
@@ -2315,8 +2318,8 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
return parseInExpression(context, canUseFullPath);
}
- if (context.castInput != null) {
- return parseCastFunction(context, canUseFullPath);
+ if (context.scalarFunctionExpression() != null) {
+ return parseScalarFunctionExpression(context.scalarFunctionExpression(), canUseFullPath);
}
if (context.functionName() != null) {
@@ -2339,14 +2342,33 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
throw new UnsupportedOperationException();
}
+ private Expression parseScalarFunctionExpression(
+ IoTDBSqlParser.ScalarFunctionExpressionContext context, boolean canUseFullPath) {
+ if (context.CAST() != null) {
+ return parseCastFunction(context, canUseFullPath);
+ } else if (context.REPLACE() != null) {
+ return parseReplaceFunction(context, canUseFullPath);
+ }
+ throw new UnsupportedOperationException();
+ }
+
private Expression parseCastFunction(
- IoTDBSqlParser.ExpressionContext castClause, boolean canUseFullPath) {
+ IoTDBSqlParser.ScalarFunctionExpressionContext castClause, boolean canUseFullPath) {
FunctionExpression functionExpression = new FunctionExpression(CAST_FUNCTION);
functionExpression.addExpression(parseExpression(castClause.castInput, canUseFullPath));
functionExpression.addAttribute(CAST_TYPE, parseAttributeValue(castClause.attributeValue()));
return functionExpression;
}
+ private Expression parseReplaceFunction(
+ IoTDBSqlParser.ScalarFunctionExpressionContext replaceClause, boolean canUseFullPath) {
+ FunctionExpression functionExpression = new FunctionExpression(REPLACE_FUNCTION);
+ functionExpression.addExpression(parseExpression(replaceClause.text, canUseFullPath));
+ functionExpression.addAttribute(REPLACE_FROM, parseStringLiteral(replaceClause.from.getText()));
+ functionExpression.addAttribute(REPLACE_TO, parseStringLiteral(replaceClause.to.getText()));
+ return functionExpression;
+ }
+
private Expression parseFunctionExpression(
IoTDBSqlParser.ExpressionContext functionClause, boolean canUseFullPath) {
FunctionExpression functionExpression =
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/column/unary/scalar/CastFunctionColumnTransformer.java b/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/column/unary/scalar/CastFunctionColumnTransformer.java
index ac761db756..6661628505 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/column/unary/scalar/CastFunctionColumnTransformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/column/unary/scalar/CastFunctionColumnTransformer.java
@@ -19,7 +19,7 @@
package org.apache.iotdb.db.mpp.transformation.dag.column.unary.scalar;
-import org.apache.iotdb.db.mpp.plan.expression.multi.builtin.helper.CastHelper;
+import org.apache.iotdb.db.mpp.plan.expression.multi.builtin.helper.CastFunctionHelper;
import org.apache.iotdb.db.mpp.transformation.dag.column.ColumnTransformer;
import org.apache.iotdb.db.mpp.transformation.dag.column.unary.UnaryColumnTransformer;
import org.apache.iotdb.tsfile.read.common.block.column.Column;
@@ -100,7 +100,7 @@ public class CastFunctionColumnTransformer extends UnaryColumnTransformer {
private void cast(ColumnBuilder columnBuilder, long value) {
switch (returnType.getTypeEnum()) {
case INT32:
- returnType.writeInt(columnBuilder, (CastHelper.castLongToInt(value)));
+ returnType.writeInt(columnBuilder, (CastFunctionHelper.castLongToInt(value)));
break;
case INT64:
returnType.writeLong(columnBuilder, value);
@@ -126,10 +126,10 @@ public class CastFunctionColumnTransformer extends UnaryColumnTransformer {
private void cast(ColumnBuilder columnBuilder, float value) {
switch (returnType.getTypeEnum()) {
case INT32:
- returnType.writeInt(columnBuilder, CastHelper.castFloatToInt(value));
+ returnType.writeInt(columnBuilder, CastFunctionHelper.castFloatToInt(value));
break;
case INT64:
- returnType.writeLong(columnBuilder, CastHelper.castFloatToLong(value));
+ returnType.writeLong(columnBuilder, CastFunctionHelper.castFloatToLong(value));
break;
case FLOAT:
returnType.writeFloat(columnBuilder, value);
@@ -152,13 +152,13 @@ public class CastFunctionColumnTransformer extends UnaryColumnTransformer {
private void cast(ColumnBuilder columnBuilder, double value) {
switch (returnType.getTypeEnum()) {
case INT32:
- returnType.writeInt(columnBuilder, CastHelper.castDoubleToInt(value));
+ returnType.writeInt(columnBuilder, CastFunctionHelper.castDoubleToInt(value));
break;
case INT64:
- returnType.writeLong(columnBuilder, CastHelper.castDoubleToLong(value));
+ returnType.writeLong(columnBuilder, CastFunctionHelper.castDoubleToLong(value));
break;
case FLOAT:
- returnType.writeFloat(columnBuilder, CastHelper.castDoubleToFloat(value));
+ returnType.writeFloat(columnBuilder, CastFunctionHelper.castDoubleToFloat(value));
break;
case DOUBLE:
returnType.writeDouble(columnBuilder, value);
@@ -211,13 +211,13 @@ public class CastFunctionColumnTransformer extends UnaryColumnTransformer {
returnType.writeLong(columnBuilder, Long.parseLong(stringValue));
break;
case FLOAT:
- returnType.writeFloat(columnBuilder, CastHelper.castTextToFloat(stringValue));
+ returnType.writeFloat(columnBuilder, CastFunctionHelper.castTextToFloat(stringValue));
break;
case DOUBLE:
- returnType.writeDouble(columnBuilder, CastHelper.castTextToDouble(stringValue));
+ returnType.writeDouble(columnBuilder, CastFunctionHelper.castTextToDouble(stringValue));
break;
case BOOLEAN:
- returnType.writeBoolean(columnBuilder, CastHelper.castTextToBoolean(stringValue));
+ returnType.writeBoolean(columnBuilder, CastFunctionHelper.castTextToBoolean(stringValue));
break;
case BINARY:
returnType.writeBinary(columnBuilder, value);
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/column/unary/scalar/ReplaceFunctionColumnTransformer.java b/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/column/unary/scalar/ReplaceFunctionColumnTransformer.java
new file mode 100644
index 0000000000..71642f9efa
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/column/unary/scalar/ReplaceFunctionColumnTransformer.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.transformation.dag.column.unary.scalar;
+
+import org.apache.iotdb.db.mpp.transformation.dag.column.ColumnTransformer;
+import org.apache.iotdb.db.mpp.transformation.dag.column.unary.UnaryColumnTransformer;
+import org.apache.iotdb.tsfile.read.common.block.column.Column;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.type.Type;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+public class ReplaceFunctionColumnTransformer extends UnaryColumnTransformer {
+ private final String from;
+ private final String to;
+
+ public ReplaceFunctionColumnTransformer(
+ Type returnType, ColumnTransformer childColumnTransformer, String from, String to) {
+ super(returnType, childColumnTransformer);
+ this.from = from;
+ this.to = to;
+ }
+
+ @Override
+ protected void doTransform(Column column, ColumnBuilder columnBuilder) {
+ for (int i = 0, n = column.getPositionCount(); i < n; i++) {
+ if (!column.isNull(i)) {
+ returnType.writeBinary(
+ columnBuilder,
+ Binary.valueOf(
+ childColumnTransformer
+ .getType()
+ .getBinary(column, i)
+ .getStringValue()
+ .replace(from, to)));
+ } else {
+ columnBuilder.appendNull();
+ }
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java b/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java
index 3b615d2e4c..888c2431b5 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java
@@ -20,7 +20,7 @@
package org.apache.iotdb.db.mpp.transformation.dag.transformer.unary.scalar;
import org.apache.iotdb.db.exception.query.QueryProcessException;
-import org.apache.iotdb.db.mpp.plan.expression.multi.builtin.helper.CastHelper;
+import org.apache.iotdb.db.mpp.plan.expression.multi.builtin.helper.CastFunctionHelper;
import org.apache.iotdb.db.mpp.transformation.api.LayerPointReader;
import org.apache.iotdb.db.mpp.transformation.dag.transformer.unary.UnaryTransformer;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -97,7 +97,7 @@ public class CastFunctionTransformer extends UnaryTransformer {
private void cast(long value) {
switch (targetDataType) {
case INT32:
- cachedInt = CastHelper.castLongToInt(value);
+ cachedInt = CastFunctionHelper.castLongToInt(value);
return;
case INT64:
cachedLong = value;
@@ -123,10 +123,10 @@ public class CastFunctionTransformer extends UnaryTransformer {
private void cast(float value) {
switch (targetDataType) {
case INT32:
- cachedInt = CastHelper.castFloatToInt(value);
+ cachedInt = CastFunctionHelper.castFloatToInt(value);
return;
case INT64:
- cachedLong = CastHelper.castFloatToLong(value);
+ cachedLong = CastFunctionHelper.castFloatToLong(value);
return;
case FLOAT:
cachedFloat = value;
@@ -149,13 +149,13 @@ public class CastFunctionTransformer extends UnaryTransformer {
private void cast(double value) {
switch (targetDataType) {
case INT32:
- cachedInt = CastHelper.castDoubleToInt(value);
+ cachedInt = CastFunctionHelper.castDoubleToInt(value);
return;
case INT64:
- cachedLong = CastHelper.castDoubleToLong(value);
+ cachedLong = CastFunctionHelper.castDoubleToLong(value);
return;
case FLOAT:
- cachedFloat = CastHelper.castDoubleToFloat(value);
+ cachedFloat = CastFunctionHelper.castDoubleToFloat(value);
return;
case DOUBLE:
cachedDouble = value;
@@ -209,13 +209,13 @@ public class CastFunctionTransformer extends UnaryTransformer {
cachedLong = Long.parseLong(stringValue);
return;
case FLOAT:
- cachedFloat = CastHelper.castTextToFloat(stringValue);
+ cachedFloat = CastFunctionHelper.castTextToFloat(stringValue);
return;
case DOUBLE:
- cachedDouble = CastHelper.castTextToDouble(stringValue);
+ cachedDouble = CastFunctionHelper.castTextToDouble(stringValue);
return;
case BOOLEAN:
- cachedBoolean = CastHelper.castTextToBoolean(stringValue);
+ cachedBoolean = CastFunctionHelper.castTextToBoolean(stringValue);
return;
case TEXT:
cachedBinary = Binary.valueOf(String.valueOf(value));
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/transformer/unary/scalar/ReplaceFunctionTransformer.java b/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/transformer/unary/scalar/ReplaceFunctionTransformer.java
new file mode 100644
index 0000000000..649b7d9616
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/transformer/unary/scalar/ReplaceFunctionTransformer.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.transformation.dag.transformer.unary.scalar;
+
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.mpp.transformation.api.LayerPointReader;
+import org.apache.iotdb.db.mpp.transformation.dag.transformer.unary.UnaryTransformer;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import java.io.IOException;
+
+public class ReplaceFunctionTransformer extends UnaryTransformer {
+ private final String from;
+ private final String to;
+
+ public ReplaceFunctionTransformer(LayerPointReader layerPointReader, String from, String to) {
+ super(layerPointReader);
+ this.from = from;
+ this.to = to;
+ }
+
+ @Override
+ public TSDataType getDataType() {
+ return TSDataType.TEXT;
+ }
+
+ @Override
+ protected void transformAndCache() throws QueryProcessException, IOException {
+ cachedBinary =
+ Binary.valueOf(layerPointReader.currentBinary().getStringValue().replace(from, to));
+ }
+}