You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/11/09 20:47:13 UTC
[incubator-pinot] branch master updated: Adding support of logical
functions AND and OR (#6249)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 8ce2271 Adding support of logical functions AND and OR (#6249)
8ce2271 is described below
commit 8ce2271f2e7b6e5cea71d4dc15814099aae88c6b
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Mon Nov 9 12:46:52 2020 -0800
Adding support of logical functions AND and OR (#6249)
---
.../common/function/TransformFunctionType.java | 3 +
.../function/AndOperatorTransformFunction.java | 48 +++++++++++++
.../function/LogicalOperatorTransformFunction.java | 79 +++++++++++++++++++++
.../function/OrOperatorTransformFunction.java | 48 +++++++++++++
.../function/TransformFunctionFactory.java | 5 ++
.../org/apache/pinot/core/util/ArrayCopyUtils.java | 6 ++
.../function/AndOperatorTransformFunctionTest.java | 34 +++++++++
.../LogicalOperatorTransformFunctionTest.java | 81 ++++++++++++++++++++++
.../function/OrOperatorTransformFunctionTest.java | 34 +++++++++
.../tests/OfflineClusterIntegrationTest.java | 28 ++++++++
10 files changed, 366 insertions(+)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java b/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
index bd47121..eabd94b 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
@@ -45,6 +45,9 @@ public enum TransformFunctionType {
LESS_THAN("less_than"),
LESS_THAN_OR_EQUAL("less_than_or_equal"),
+ AND("and"),
+ OR("or"),
+
CAST("cast"),
CASE("case"),
JSONEXTRACTSCALAR("jsonExtractScalar"),
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/AndOperatorTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/AndOperatorTransformFunction.java
new file mode 100644
index 0000000..95dad8d
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/AndOperatorTransformFunction.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import org.apache.pinot.common.function.TransformFunctionType;
+
+
+/**
+ * The <code>AndOperatorTransformFunction</code> extends <code>LogicalOperatorTransformFunction</code> to
+ * implement the logical operator 'AND'.
+ *
+ * The results are in boolean format and stored as an integer array with 1 represents true and 0 represents false.
+ *
+ * SQL Syntax:
+ * exprA AND exprB
+ *
+ */
+public class AndOperatorTransformFunction extends LogicalOperatorTransformFunction {
+
+ @Override
+ public String getName() {
+ return TransformFunctionType.AND.getName();
+ }
+
+ @Override
+ int getLogicalFuncResult(int arg1, int arg2) {
+ if ((arg1 != 0) && (arg2 != 0)) {
+ return 1;
+ }
+ return 0;
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java
new file mode 100644
index 0000000..59e5c46
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import com.google.common.base.Preconditions;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import org.apache.pinot.core.common.DataSource;
+import org.apache.pinot.core.operator.blocks.ProjectionBlock;
+import org.apache.pinot.core.operator.transform.TransformResultMetadata;
+import org.apache.pinot.core.plan.DocIdSetPlanNode;
+import org.apache.pinot.core.util.ArrayCopyUtils;
+
+
+/**
+ * <code>LogicalOperatorTransformFunction</code> abstracts common functions for logical operators (AND, OR).
+ * The results are in boolean format and stored as an integer array with 1 represents true and 0 represents false.
+ */
+public abstract class LogicalOperatorTransformFunction extends BaseTransformFunction {
+
+ protected List<TransformFunction> _arguments;
+ protected int[] _results;
+
+ @Override
+ public void init(List<TransformFunction> arguments, Map<String, DataSource> dataSourceMap) {
+ _arguments = arguments;
+ Preconditions.checkState(arguments.size() > 1, String
+ .format("Expect more than 1 argument for logical operator [%s], args [%s].", getName(),
+ Arrays.toString(arguments.toArray())));
+ for (TransformFunction argument : arguments) {
+ Preconditions.checkState(
+ argument.getResultMetadata().getDataType().isNumeric() && argument.getResultMetadata().isSingleValue(), String
+ .format(
+ "Unsupported data type for logical operator [%s] arguments, only supports single-valued number. Invalid argument: expression [%s], result type [%s]",
+ getName(), argument.getName(), argument.getResultMetadata()));
+ }
+ }
+
+ @Override
+ public TransformResultMetadata getResultMetadata() {
+ return INT_SV_NO_DICTIONARY_METADATA;
+ }
+
+ @Override
+ public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
+ if (_results == null) {
+ _results = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int length = projectionBlock.getNumDocs();
+ ArrayCopyUtils.copy(_arguments.get(0).transformToIntValuesSV(projectionBlock), _results, length);
+ for (int i = 1; i < _arguments.size(); i++) {
+ final TransformFunction transformFunction = _arguments.get(i);
+ int[] results = transformFunction.transformToIntValuesSV(projectionBlock);
+ for (int j = 0; j < length; j++) {
+ _results[j] = getLogicalFuncResult(_results[j], results[j]);
+ }
+ }
+ return _results;
+ }
+
+ abstract int getLogicalFuncResult(int arg1, int arg2);
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/OrOperatorTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/OrOperatorTransformFunction.java
new file mode 100644
index 0000000..01a9706
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/OrOperatorTransformFunction.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import org.apache.pinot.common.function.TransformFunctionType;
+
+
+/**
+ * The <code>OrOperatorTransformFunction</code> extends <code>LogicalOperatorTransformFunction</code> to
+ * implement the logical operator 'OR'.
+ *
+ * The results are in boolean format and stored as an integer array with 1 represents true and 0 represents false.
+ *
+ * SQL Syntax:
+ * exprA OR exprB
+ *
+ */
+public class OrOperatorTransformFunction extends LogicalOperatorTransformFunction {
+
+ @Override
+ public String getName() {
+ return TransformFunctionType.OR.getName();
+ }
+
+ @Override
+ int getLogicalFuncResult(int arg1, int arg2) {
+ if ((arg1 == 0) && (arg2 == 0)) {
+ return 0;
+ }
+ return 1;
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
index cf1685f..3d6ff94 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
@@ -109,6 +109,11 @@ public class TransformFunctionFactory {
GreaterThanOrEqualTransformFunction.class);
put(canonicalize(TransformFunctionType.LESS_THAN.getName().toLowerCase()), LessThanTransformFunction.class);
put(canonicalize(TransformFunctionType.LESS_THAN_OR_EQUAL.getName().toLowerCase()), LessThanOrEqualTransformFunction.class);
+
+ // logical functions
+ put(canonicalize(TransformFunctionType.AND.getName().toLowerCase()), AndOperatorTransformFunction.class);
+ put(canonicalize(TransformFunctionType.OR.getName().toLowerCase()), OrOperatorTransformFunction.class);
+
// geo functions
// geo constructors
put(canonicalize(TransformFunctionType.ST_GEOG_FROM_TEXT.getName().toLowerCase()), StGeogFromTextFunction.class);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/util/ArrayCopyUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/util/ArrayCopyUtils.java
index d88c02c..0df2e5c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/util/ArrayCopyUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/util/ArrayCopyUtils.java
@@ -29,6 +29,12 @@ public class ArrayCopyUtils {
private ArrayCopyUtils() {
}
+ public static void copy(int[] src, int[] dest, int length) {
+ for (int i = 0; i < length; i++) {
+ dest[i] = src[i];
+ }
+ }
+
public static void copy(int[] src, long[] dest, int length) {
for (int i = 0; i < length; i++) {
dest[i] = src[i];
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/AndOperatorTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/AndOperatorTransformFunctionTest.java
new file mode 100644
index 0000000..7f0e854
--- /dev/null
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/AndOperatorTransformFunctionTest.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+public class AndOperatorTransformFunctionTest extends LogicalOperatorTransformFunctionTest {
+ @Override
+ int getExpectedValue(boolean arg1, boolean arg2) {
+ if (arg1 && arg2) {
+ return 1;
+ }
+ return 0;
+ }
+
+ @Override
+ String getFuncName() {
+ return new AndOperatorTransformFunction().getName();
+ }
+}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunctionTest.java
new file mode 100644
index 0000000..15b13e4
--- /dev/null
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunctionTest.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.pinot.common.request.Expression;
+import org.apache.pinot.common.utils.StringUtil;
+import org.apache.pinot.core.query.exception.BadQueryRequestException;
+import org.apache.pinot.core.query.request.context.ExpressionContext;
+import org.apache.pinot.core.query.request.context.FunctionContext;
+import org.apache.pinot.core.query.request.context.utils.QueryContextConverterUtils;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+
+/**
+ * LogicalOperatorTransformFunctionTest abstracts common test methods for:
+ * AndTransformFunctionTest
+ * OrTransformFunctionTest
+ *
+ */
+public abstract class LogicalOperatorTransformFunctionTest extends BaseTransformFunctionTest {
+
+ abstract int getExpectedValue(boolean arg1, boolean arg2);
+
+ abstract String getFuncName();
+
+ @Test
+ public void testLogicalOperatorTransformFunction() {
+ ExpressionContext intEqualsExpr =
+ QueryContextConverterUtils.getExpression(String.format("EQUALS(%s, %d)", INT_SV_COLUMN, _intSVValues[0]));
+ ExpressionContext longEqualsExpr =
+ QueryContextConverterUtils.getExpression(String.format("EQUALS(%s, %d)", LONG_SV_COLUMN, _longSVValues[0]));
+ ExpressionContext expression = ExpressionContext.forFunction(
+ new FunctionContext(FunctionContext.Type.TRANSFORM, getFuncName(),
+ Arrays.asList(intEqualsExpr, longEqualsExpr)));
+ TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap);
+ Assert.assertEquals(transformFunction.getName(), getFuncName().toLowerCase());
+ int[] expectedIntValues = new int[NUM_ROWS];
+ for (int i = 0; i < NUM_ROWS; i++) {
+ expectedIntValues[i] = getExpectedValue(_intSVValues[i] == _intSVValues[0], _longSVValues[i] == _longSVValues[0]);
+ }
+ testTransformFunction(transformFunction, expectedIntValues);
+ }
+
+ @Test(dataProvider = "testIllegalArguments", expectedExceptions = {BadQueryRequestException.class})
+ public void testIllegalArguments(String[] expressions) {
+ List<ExpressionContext> expressionContextList = new ArrayList<>();
+ for (int i = 0; i < expressions.length; i++) {
+ expressionContextList.add(QueryContextConverterUtils.getExpression(expressions[i]));
+ }
+ TransformFunctionFactory.get(ExpressionContext
+ .forFunction(new FunctionContext(FunctionContext.Type.TRANSFORM, getFuncName(), expressionContextList)),
+ _dataSourceMap);
+ }
+
+ @DataProvider(name = "testIllegalArguments")
+ public Object[][] testIllegalArguments() {
+ String intEqualsExpr = String.format("EQUALS(%s, %d)", INT_SV_COLUMN, _intSVValues[0]);
+ return new Object[][]{new Object[]{intEqualsExpr}, new Object[]{intEqualsExpr, STRING_SV_COLUMN}};
+ }
+}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/OrOperatorTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/OrOperatorTransformFunctionTest.java
new file mode 100644
index 0000000..a137b49
--- /dev/null
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/OrOperatorTransformFunctionTest.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+public class OrOperatorTransformFunctionTest extends LogicalOperatorTransformFunctionTest {
+ @Override
+ int getExpectedValue(boolean arg1, boolean arg2) {
+ if (arg1 || arg2) {
+ return 1;
+ }
+ return 0;
+ }
+
+ @Override
+ String getFuncName() {
+ return new OrOperatorTransformFunction().getName();
+ }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
index a33d51d..52e0049 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
@@ -992,6 +992,34 @@ public class OfflineClusterIntegrationTest extends BaseClusterIntegrationTestSet
}
@Test
+ public void testCaseStatementWithLogicalTransformFunction()
+ throws Exception {
+ String sqlQuery =
+ "SELECT ArrDelay"
+ + ", CASE WHEN ArrDelay > 50 OR ArrDelay < 10 THEN 10 ELSE 0 END"
+ + ", CASE WHEN ArrDelay < 50 AND ArrDelay >= 10 THEN 10 ELSE 0 END"
+ + " FROM mytable LIMIT 1000";
+ JsonNode response = postSqlQuery(sqlQuery, _brokerBaseApiUrl);
+ JsonNode rows = response.get("resultTable").get("rows");
+ assertEquals(response.get("exceptions").size(), 0);
+ for (int i = 0; i < rows.size(); i++) {
+ int row0 = rows.get(i).get(0).asInt();
+ int row1 = rows.get(i).get(1).asInt();
+ int row2 = rows.get(i).get(2).asInt();
+ if (row0 > 50 || row0 < 10) {
+ assertEquals(row1, 10);
+ } else {
+ assertEquals(row1, 0);
+ }
+ if (row0 < 50 && row0 >= 10) {
+ assertEquals(row2, 10);
+ } else {
+ assertEquals(row2, 0);
+ }
+ }
+ }
+
+ @Test
public void testCaseStatementWithInAggregation()
throws Exception {
testCountVsCaseQuery("origin = 'ATL'");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org