You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ki...@apache.org on 2020/05/24 01:49:43 UTC
[incubator-pinot] branch generic-transform-function updated
(9d16600 -> f599355)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a change to branch generic-transform-function
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard 9d16600 Adding support to invoke any scalar function via GenericTransformFunction
add 5e49f27 Make Compilation time function evaluation after query parser (#5437)
new f599355 Adding support to invoke any scalar function via GenericTransformFunction
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (9d16600)
\
N -- N -- N refs/heads/generic-transform-function (f599355)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../pinot/common/function/FunctionRegistry.java | 13 ++-
.../pinot/common/function/StringFunctions.java | 10 ++-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 92 +++++++++++++---------
.../pinot/sql/parsers/CalciteSqlCompilerTest.java | 49 +++++++++++-
.../function/GenericTransformFunction.java | 12 +--
5 files changed, 123 insertions(+), 53 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Adding support to invoke any scalar
function via GenericTransformFunction
Posted by ki...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch generic-transform-function
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit f5993552b7b29c54c4b796a8218e22cdd1381842
Author: kishoreg <g....@gmail.com>
AuthorDate: Sat May 23 10:49:34 2020 -0700
Adding support to invoke any scalar function via GenericTransformFunction
---
.../pinot/common/function/FunctionRegistry.java | 10 +-
.../function/GenericTransformFunction.java | 139 +++++++++++++++++++++
.../function/TransformFunctionFactory.java | 20 ++-
3 files changed, 163 insertions(+), 6 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionRegistry.java b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionRegistry.java
index d9f7a79..dcecf73 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionRegistry.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionRegistry.java
@@ -53,8 +53,12 @@ public class FunctionRegistry {
}
public static void registerFunction(Method method) {
- FunctionInfo functionInfo = new FunctionInfo(method, method.getDeclaringClass());
- _functionInfoMap.put(method.getName().toLowerCase(), functionInfo);
+ try {
+ FunctionInfo functionInfo = new FunctionInfo(method, method.getDeclaringClass());
+ _functionInfoMap.put(method.getName().toLowerCase(), functionInfo);
+ }catch (Throwable t){
+
+ }
}
public static boolean containsFunctionByName(String funcName) {
@@ -63,6 +67,7 @@ public class FunctionRegistry {
static {
try {
+ FunctionRegistry.registerFunction(StringFunctions.class.getDeclaredMethod("reverse", String.class));
FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochSeconds", Long.class));
FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochMinutes", Long.class));
FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochHours", Long.class));
@@ -92,7 +97,6 @@ public class FunctionRegistry {
FunctionRegistry.registerFunction(StringFunctions.class.getDeclaredMethod("reverse", String.class));
} catch (NoSuchMethodException e) {
LOGGER.error("Caught exception when registering function", e);
- throw new IllegalStateException(e);
}
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GenericTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GenericTransformFunction.java
new file mode 100644
index 0000000..c3ffb79
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GenericTransformFunction.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import org.apache.pinot.common.function.FunctionInfo;
+import org.apache.pinot.common.function.FunctionInvoker;
+import org.apache.pinot.core.common.DataSource;
+import org.apache.pinot.core.operator.blocks.ProjectionBlock;
+import org.apache.pinot.core.operator.transform.TransformResultMetadata;
+import org.apache.pinot.core.plan.DocIdSetPlanNode;
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+public class GenericTransformFunction extends BaseTransformFunction {
+
+ private FunctionInfo _info;
+ FunctionInvoker _functionInvoker;
+ String _name;
+ Object[] _args;
+ List<Integer> _nonLiteralArgIndices;
+ List<FieldSpec.DataType> _nonLiteralArgType;
+ List<TransformFunction> _nonLiteralTransformFunction;
+ String[] _stringResult;
+
+ public GenericTransformFunction() {
+ _nonLiteralArgIndices = new ArrayList();
+ _nonLiteralArgType = new ArrayList();
+ _nonLiteralTransformFunction = new ArrayList<>();
+ }
+
+ @Override
+ public String getName() {
+ return _name;
+ }
+
+ public void setFunction(String functionName, FunctionInfo info)
+ throws Exception {
+ _name = functionName;
+ _info = info;
+ _functionInvoker = new FunctionInvoker(info);
+ }
+
+ @Override
+ public void init(List<TransformFunction> arguments, Map<String, DataSource> dataSourceMap) {
+ //assert method.args.length == arguments.size
+ _args = new Object[arguments.size()];
+
+ for (int i = 0; i < arguments.size(); i++) {
+ TransformFunction function = arguments.get(i);
+ if (function instanceof LiteralTransformFunction) {
+ String literal = ((LiteralTransformFunction) function).getLiteral();
+ //convert String to the right dataType based on method param
+
+ Class paramType = _functionInvoker.getParameterTypes()[i];
+ switch (paramType.getTypeName().toLowerCase()) {
+ case "Integer":
+ _args[i] = Integer.parseInt(literal);
+ break;
+ case "java.lang.string":
+ _args[i] = literal;
+ break;
+ //add other types and throw exception for non primitive/string classes
+ }
+ } else {
+ _nonLiteralArgIndices.add(i);
+ _nonLiteralTransformFunction.add(function);
+ Class paramType = _functionInvoker.getParameterTypes()[i];
+ //find the right pinot data Type
+ switch (paramType.getTypeName().toLowerCase()) {
+ case "integer":
+ _nonLiteralArgType.add(FieldSpec.DataType.INT);
+ break;
+ case "java.lang.string":
+ _nonLiteralArgType.add(FieldSpec.DataType.STRING);
+ break;
+ //todo add other types
+ }
+ }
+ }
+ }
+
+ @Override
+ public TransformResultMetadata getResultMetadata() {
+ return STRING_SV_NO_DICTIONARY_METADATA;
+ }
+
+ @SuppressWarnings("Duplicates")
+ @Override
+ public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) {
+ if (_stringResult == null) {
+ _stringResult = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+
+ int length = projectionBlock.getNumDocs();
+ int numNonLiteralArgs = _nonLiteralArgIndices.size();
+ Object[][] nonLiteralBlockValues = new Object[numNonLiteralArgs][];
+
+ for (int i = 0; i < numNonLiteralArgs; i++) {
+ TransformFunction transformFunc = _nonLiteralTransformFunction.get(i);
+ FieldSpec.DataType returnType = _nonLiteralArgType.get(i);
+ switch (returnType) {
+ case STRING:
+ nonLiteralBlockValues[i] = transformFunc.transformToStringValuesSV(projectionBlock);
+ //todo handle other types
+ }
+ }
+
+ //now invoke the actual function
+ for (int i = 0; i < length; i++) {
+ for (int k = 0; k < numNonLiteralArgs; k++) {
+ _args[_nonLiteralArgIndices.get(k)] = nonLiteralBlockValues[k][i];
+ }
+ _stringResult[i] = (String) _functionInvoker.process(_args);
+ }
+ return _stringResult;
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
index e9bd6bc..aac1938 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
@@ -23,6 +23,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.pinot.common.function.FunctionInfo;
+import org.apache.pinot.common.function.FunctionRegistry;
import org.apache.pinot.common.function.TransformFunctionType;
import org.apache.pinot.common.request.transform.TransformExpressionTree;
import org.apache.pinot.core.common.DataSource;
@@ -65,7 +67,8 @@ public class TransformFunctionFactory {
put(TransformFunctionType.SQRT.getName().toLowerCase(), SqrtTransformFunction.class);
put(TransformFunctionType.CAST.getName().toLowerCase(), CastTransformFunction.class);
- put(TransformFunctionType.JSONEXTRACTSCALAR.getName().toLowerCase(), JsonExtractScalarTransformFunction.class);
+ put(TransformFunctionType.JSONEXTRACTSCALAR.getName().toLowerCase(),
+ JsonExtractScalarTransformFunction.class);
put(TransformFunctionType.JSONEXTRACTKEY.getName().toLowerCase(), JsonExtractKeyTransformFunction.class);
put(TransformFunctionType.TIMECONVERT.getName().toLowerCase(), TimeConversionTransformFunction.class);
put(TransformFunctionType.DATETIMECONVERT.getName().toLowerCase(), DateTimeConversionTransformFunction.class);
@@ -112,13 +115,24 @@ public class TransformFunctionFactory {
switch (expression.getExpressionType()) {
case FUNCTION:
String functionName = expression.getValue();
- Class<? extends TransformFunction> transformFunctionClass = TRANSFORM_FUNCTION_MAP.get(functionName);
+ Class<? extends TransformFunction> transformFunctionClass;
+ FunctionInfo functionInfo = null;
+ if (FunctionRegistry.containsFunctionByName(functionName)) {
+ transformFunctionClass = GenericTransformFunction.class;
+ functionInfo = FunctionRegistry.getFunctionByName(functionName);
+ } else {
+ transformFunctionClass = TRANSFORM_FUNCTION_MAP.get(functionName);
+ }
+
if (transformFunctionClass == null) {
throw new BadQueryRequestException("Unsupported transform function: " + functionName);
}
try {
transformFunction = transformFunctionClass.newInstance();
- } catch (InstantiationException | IllegalAccessException e) {
+ if (functionInfo != null) {
+ ((GenericTransformFunction) transformFunction).setFunction(functionName, functionInfo);
+ }
+ } catch (Exception e) {
throw new RuntimeException("Caught exception while instantiating transform function: " + functionName, e);
}
List<TransformExpressionTree> children = expression.getChildren();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org