You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/05/23 05:18:55 UTC
[incubator-pinot] branch master updated: Adding support to execute
functions during query compilation (#5406)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 3f8ba71 Adding support to execute functions during query compilation (#5406)
3f8ba71 is described below
commit 3f8ba7104869be4fcda5080631d546a5d7b30791
Author: Kishore Gopalakrishna <g....@gmail.com>
AuthorDate: Fri May 22 22:18:41 2020 -0700
Adding support to execute functions during query compilation (#5406)
* Adding support to execute functions during query compilation
* Remove function formatDatetime
Co-authored-by: Xiang Fu <fx...@gmail.com>
---
.../pinot/common}/function/DateTimeFunctions.java | 10 ++-
.../common}/function/DateTimePatternHandler.java | 2 +-
.../pinot/common}/function/FunctionInfo.java | 2 +-
.../pinot/common}/function/FunctionInvoker.java | 8 +-
.../pinot/common/function/FunctionRegistry.java | 96 ++++++++++++++++++++++
.../pinot/common}/function/JsonFunctions.java | 2 +-
.../pinot/common/utils/request/RequestUtils.java | 23 ++++++
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 91 +++++++++++++++-----
.../pinot/sql/parsers/CalciteSqlCompilerTest.java | 49 +++++++++--
.../pinot/core/data/function/FunctionRegistry.java | 84 -------------------
.../data/function/InbuiltFunctionEvaluator.java | 3 +
.../function/DateTimeFunctionEvaluatorTest.java | 1 +
.../function/InbuiltFunctionEvaluatorTest.java | 2 +
13 files changed, 258 insertions(+), 115 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimeFunctions.java b/pinot-common/src/main/java/org/apache/pinot/common/function/DateTimeFunctions.java
similarity index 97%
rename from pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimeFunctions.java
rename to pinot-common/src/main/java/org/apache/pinot/common/function/DateTimeFunctions.java
index a9e636a..c0ebe1b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimeFunctions.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/DateTimeFunctions.java
@@ -16,9 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.data.function;
+package org.apache.pinot.common.function;
import java.util.concurrent.TimeUnit;
+import org.joda.time.format.DateTimeFormat;
/**
@@ -216,4 +217,11 @@ public class DateTimeFunctions {
static Long fromDateTime(String dateTimeString, String pattern) {
return DateTimePatternHandler.parseDateTimeStringToEpochMillis(dateTimeString, pattern);
}
+
+ /**
+ * Return current time as epoch millis
+ */
+ static Long now() {
+ return System.currentTimeMillis();
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimePatternHandler.java b/pinot-common/src/main/java/org/apache/pinot/common/function/DateTimePatternHandler.java
similarity index 97%
rename from pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimePatternHandler.java
rename to pinot-common/src/main/java/org/apache/pinot/common/function/DateTimePatternHandler.java
index e94d700..c9c0f81 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimePatternHandler.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/DateTimePatternHandler.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.data.function;
+package org.apache.pinot.common.function;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionInfo.java b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionInfo.java
similarity index 98%
rename from pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionInfo.java
rename to pinot-common/src/main/java/org/apache/pinot/common/function/FunctionInfo.java
index c79ff34..5cb81eb 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionInfo.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionInfo.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.data.function;
+package org.apache.pinot.common.function;
import java.lang.reflect.Method;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionInvoker.java b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionInvoker.java
similarity index 92%
rename from pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionInvoker.java
rename to pinot-common/src/main/java/org/apache/pinot/common/function/FunctionInvoker.java
index 5ce876f..73c12ed 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionInvoker.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionInvoker.java
@@ -16,8 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.data.function;
+package org.apache.pinot.common.function;
+import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
@@ -49,11 +50,14 @@ public class FunctionInvoker {
throws Exception {
_functionInfo = functionInfo;
_method = functionInfo.getMethod();
+ _method.setAccessible(true);
Class<?> clazz = functionInfo.getClazz();
if (Modifier.isStatic(_method.getModifiers())) {
_instance = null;
} else {
- _instance = clazz.newInstance();
+ Constructor<?> constructor = clazz.getDeclaredConstructor();
+ constructor.setAccessible(true);
+ _instance = constructor.newInstance();
}
}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionRegistry.java b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionRegistry.java
new file mode 100644
index 0000000..a502bf2
--- /dev/null
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionRegistry.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.function;
+
+import com.google.common.base.Preconditions;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Registry for inbuilt Pinot functions
+ */
+public class FunctionRegistry {
+ private static final Logger LOGGER = LoggerFactory.getLogger(FunctionRegistry.class);
+ private static final Map<String, FunctionInfo> _functionInfoMap = new HashMap<>();
+
+ /**
+ * Given a function name, asserts that a corresponding function was registered during construction and returns it
+ */
+ public static FunctionInfo getFunctionByName(String functionName) {
+ Preconditions.checkArgument(_functionInfoMap.containsKey(functionName.toLowerCase()));
+ return _functionInfoMap.get(functionName.toLowerCase());
+ }
+
+ /**
+ * Given a function name and a set of argument types, asserts that a corresponding function
+ * was registered during construction and returns it
+ */
+ public static FunctionInfo getFunctionByNameWithApplicableArgumentTypes(String functionName,
+ Class<?>[] argumentTypes) {
+ FunctionInfo functionInfo = getFunctionByName(functionName);
+ Preconditions.checkArgument(functionInfo.isApplicable(argumentTypes));
+ return functionInfo;
+ }
+
+ public static void registerFunction(Method method) {
+ FunctionInfo functionInfo = new FunctionInfo(method, method.getDeclaringClass());
+ _functionInfoMap.put(method.getName().toLowerCase(), functionInfo);
+ }
+
+ public static boolean containsFunctionByName(String funcName) {
+ return _functionInfoMap.containsKey(funcName.toLowerCase());
+ }
+
+ static {
+ try {
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochSeconds", Long.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochMinutes", Long.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochHours", Long.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochDays", Long.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochSecondsRounded", Long.class, Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochMinutesRounded", Long.class, Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochHoursRounded", Long.class, Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochDaysRounded", Long.class, Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochSecondsBucket", Long.class, Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochMinutesBucket", Long.class, Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochHoursBucket", Long.class, Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochDaysBucket", Long.class, Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochSeconds", Long.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochMinutes", Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochHours", Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochDays", Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochSecondsBucket", Long.class, Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochMinutesBucket", Number.class, Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochHoursBucket", Number.class, Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochDaysBucket", Number.class, Number.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("toDateTime", Long.class, String.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromDateTime", String.class, String.class));
+ FunctionRegistry.registerFunction(DateTimeFunctions.class.getDeclaredMethod("now"));
+
+ FunctionRegistry.registerFunction(JsonFunctions.class.getDeclaredMethod("toJsonMapStr", Map.class));
+ } catch (NoSuchMethodException e) {
+ LOGGER.error("Caught exception when registering function", e);
+ throw new IllegalStateException(e);
+ }
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/function/JsonFunctions.java b/pinot-common/src/main/java/org/apache/pinot/common/function/JsonFunctions.java
similarity index 96%
rename from pinot-core/src/main/java/org/apache/pinot/core/data/function/JsonFunctions.java
rename to pinot-common/src/main/java/org/apache/pinot/common/function/JsonFunctions.java
index 84b7446..6ca3ae6 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/function/JsonFunctions.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/JsonFunctions.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.core.data.function;
+package org.apache.pinot.common.function;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.util.Map;
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/request/RequestUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/request/RequestUtils.java
index d1e3eda..cc4b9c1 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/request/RequestUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/request/RequestUtils.java
@@ -50,6 +50,7 @@ import org.apache.pinot.pql.parsers.pql2.ast.IntegerLiteralAstNode;
import org.apache.pinot.pql.parsers.pql2.ast.LiteralAstNode;
import org.apache.pinot.pql.parsers.pql2.ast.PredicateAstNode;
import org.apache.pinot.pql.parsers.pql2.ast.StringLiteralAstNode;
+import org.apache.pinot.sql.parsers.SqlCompilationException;
public class RequestUtils {
@@ -159,6 +160,28 @@ public class RequestUtils {
return expression;
}
+ public static Expression getLiteralExpression(Object object) {
+ if (object instanceof Integer) {
+ return RequestUtils.getLiteralExpression((Integer) object);
+ }
+ if (object instanceof Long) {
+ return RequestUtils.getLiteralExpression((Long) object);
+ }
+ if (object instanceof Float) {
+ return RequestUtils.getLiteralExpression((Float) object);
+ }
+ if (object instanceof Double) {
+ return RequestUtils.getLiteralExpression((Double) object);
+ }
+ if (object instanceof String) {
+ return RequestUtils.getLiteralExpression((String) object);
+ }
+ if(object instanceof SqlLiteral) {
+ return RequestUtils.getLiteralExpression((SqlLiteral) object);
+ }
+ throw new SqlCompilationException(new IllegalArgumentException("Unsupported Literal value type - " + object.getClass()));
+ }
+
public static Expression getFunctionExpression(String operator) {
Expression expression = new Expression(ExpressionType.FUNCTION);
Function function = new Function(operator);
diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
index 5abffc3..a672fc8 100644
--- a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
+++ b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
@@ -45,6 +45,9 @@ import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl;
import org.apache.calcite.sql.validate.SqlConformanceEnum;
import org.apache.pinot.common.function.AggregationFunctionType;
import org.apache.pinot.common.function.FunctionDefinitionRegistry;
+import org.apache.pinot.common.function.FunctionInfo;
+import org.apache.pinot.common.function.FunctionInvoker;
+import org.apache.pinot.common.function.FunctionRegistry;
import org.apache.pinot.common.request.DataSource;
import org.apache.pinot.common.request.Expression;
import org.apache.pinot.common.request.ExpressionType;
@@ -66,7 +69,7 @@ public class CalciteSqlParser {
* or not they quoted; after which, identifiers are matched
* case-insensitively. Double quotes allow identifiers to contain
* non-alphanumeric characters. */
- private static Lex PINOT_LEX = Lex.MYSQL_ANSI;
+ private static final Lex PINOT_LEX = Lex.MYSQL_ANSI;
// To Keep the backward compatibility with 'OPTION' Functionality in PQL, which is used to
// provide more hints for query processing.
@@ -593,29 +596,77 @@ public class CalciteSqlParser {
// Move on to process default logic.
}
default:
- SqlBasicCall funcSqlNode = (SqlBasicCall) node;
- String funcName = funcSqlNode.getOperator().getKind().name();
- if (funcSqlNode.getOperator().getKind() == SqlKind.OTHER_FUNCTION) {
- funcName = funcSqlNode.getOperator().getName();
+ return evaluateFunctionExpression((SqlBasicCall) node);
+ }
+ }
+
+ private static String extractFunctionName(SqlBasicCall funcSqlNode) {
+ String funcName = funcSqlNode.getOperator().getKind().name();
+ if (funcSqlNode.getOperator().getKind() == SqlKind.OTHER_FUNCTION) {
+ funcName = funcSqlNode.getOperator().getName();
+ }
+ if (funcName.equalsIgnoreCase(SqlKind.COUNT.toString()) && (funcSqlNode.getFunctionQuantifier() != null)
+ && funcSqlNode.getFunctionQuantifier().toValue()
+ .equalsIgnoreCase(AggregationFunctionType.DISTINCT.getName())) {
+ funcName = AggregationFunctionType.DISTINCTCOUNT.getName();
+ }
+ return funcName;
+ }
+
+ private static Expression evaluateFunctionExpression(SqlBasicCall funcSqlNode) {
+ String funcName = extractFunctionName(funcSqlNode);
+ Expression funcExpr = RequestUtils.getFunctionExpression(funcName);
+ if (FunctionRegistry.containsFunctionByName(funcName) && isCompileTimeEvaluationPossible(funcExpr)) {
+ int functionOperandsLength = funcSqlNode.getOperands().length;
+ FunctionInfo functionInfo = FunctionRegistry.getFunctionByName(funcName);
+ Object[] arguments = new Object[functionOperandsLength];
+ for (int i = 0; i < functionOperandsLength; i++) {
+ if (funcSqlNode.getOperands()[i] instanceof SqlLiteral) {
+ arguments[i] = ((SqlLiteral) funcSqlNode.getOperands()[i]).toValue();
+ } else {
+ // Evaluate function call (SqlBasicCall) recursively.
+ arguments[i] = evaluateFunctionExpression((SqlBasicCall) funcSqlNode.getOperands()[i]).getLiteral().getFieldValue();
}
- if (funcName.equalsIgnoreCase(SqlKind.COUNT.toString()) && (funcSqlNode.getFunctionQuantifier() != null)
- && funcSqlNode.getFunctionQuantifier().toValue()
- .equalsIgnoreCase(AggregationFunctionType.DISTINCT.getName())) {
- funcName = AggregationFunctionType.DISTINCTCOUNT.getName();
+ }
+ try {
+ FunctionInvoker invoker = new FunctionInvoker(functionInfo);
+ Object result = invoker.process(arguments);
+ return RequestUtils.getLiteralExpression(result);
+ } catch (Exception e) {
+ throw new SqlCompilationException(new IllegalArgumentException("Unsupported function - " + funcName, e));
+ }
+ }
+ for (SqlNode child : funcSqlNode.getOperands()) {
+ if (child instanceof SqlNodeList) {
+ final Iterator<SqlNode> iterator = ((SqlNodeList) child).iterator();
+ while (iterator.hasNext()) {
+ final SqlNode next = iterator.next();
+ funcExpr.getFunctionCall().addToOperands(toExpression(next));
}
- final Expression funcExpr = RequestUtils.getFunctionExpression(funcName);
- for (SqlNode child : funcSqlNode.getOperands()) {
- if (child instanceof SqlNodeList) {
- final Iterator<SqlNode> iterator = ((SqlNodeList) child).iterator();
- while (iterator.hasNext()) {
- final SqlNode next = iterator.next();
- funcExpr.getFunctionCall().addToOperands(toExpression(next));
- }
- } else {
- funcExpr.getFunctionCall().addToOperands(toExpression(child));
+ } else {
+ funcExpr.getFunctionCall().addToOperands(toExpression(child));
+ }
+ }
+ return funcExpr;
+ }
+ /**
+ * Utility method to check if the function can be evaluated during the query compilation phae
+ * @param funcExpr
+ * @return true if all arguments are literals
+ */
+ private static boolean isCompileTimeEvaluationPossible(Expression funcExpr) {
+ Function functionCall = funcExpr.getFunctionCall();
+ if (functionCall.getOperandsSize() > 0) {
+ for (Expression expression : functionCall.getOperands()) {
+ if (expression.getType() == ExpressionType.FUNCTION) {
+ if (!isCompileTimeEvaluationPossible(expression)){
+ return false;
}
+ } else if (expression.getType() != ExpressionType.LITERAL) {
+ return false;
}
- return funcExpr;
+ }
}
+ return true;
}
}
diff --git a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
index bebdc9a..ff92f2e 100644
--- a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
@@ -21,6 +21,9 @@ package org.apache.pinot.sql.parsers;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.List;
import org.apache.calcite.sql.SqlKind;
@@ -1470,18 +1473,54 @@ public class CalciteSqlCompilerTest {
@Test
public void testNoArgFunction() {
+ String query = "SELECT noArgFunc() FROM foo ";
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
+ Assert.assertEquals(pinotQuery.getSelectList().get(0).getFunctionCall().getOperator(), "noArgFunc");
+
+ query = "SELECT a FROM foo where time_col > noArgFunc()";
+ pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
+ Function greaterThan = pinotQuery.getFilterExpression().getFunctionCall();
+ Function minus = greaterThan.getOperands().get(0).getFunctionCall();
+ Assert.assertEquals(minus.getOperands().get(1).getFunctionCall().getOperator(), "noArgFunc");
+
+ query = "SELECT sum(a), noArgFunc() FROM foo group by noArgFunc()";
+ pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
+ Assert.assertEquals(pinotQuery.getGroupByList().get(0).getFunctionCall().getOperator(), "noArgFunc");
+ }
+
+ @Test
+ public void testCompilationInvokedFunction() {
String query = "SELECT now() FROM foo ";
+ long lowerBound = System.currentTimeMillis();
PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
- Assert.assertEquals(pinotQuery.getSelectList().get(0).getFunctionCall().getOperator(), "now");
+ long nowTs = pinotQuery.getSelectList().get(0).getLiteral().getLongValue();
+ long upperBound = System.currentTimeMillis();
+ Assert.assertTrue(nowTs >= lowerBound);
+ Assert.assertTrue(nowTs <= upperBound);
query = "SELECT a FROM foo where time_col > now()";
+ lowerBound = System.currentTimeMillis();
pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
Function greaterThan = pinotQuery.getFilterExpression().getFunctionCall();
- Function minus = greaterThan.getOperands().get(0).getFunctionCall();
- Assert.assertEquals(minus.getOperands().get(1).getFunctionCall().getOperator(), "now");
+ nowTs = greaterThan.getOperands().get(1).getLiteral().getLongValue();
+ upperBound = System.currentTimeMillis();
+ Assert.assertTrue(nowTs >= lowerBound);
+ Assert.assertTrue(nowTs <= upperBound);
- query = "SELECT sum(a), now() FROM foo group by now()";
+ query = "SELECT a FROM foo where time_col > fromDateTime('2020-01-01 UTC', 'yyyy-MM-dd z')";
pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
- Assert.assertEquals(pinotQuery.getGroupByList().get(0).getFunctionCall().getOperator(), "now");
+ greaterThan = pinotQuery.getFilterExpression().getFunctionCall();
+ nowTs = greaterThan.getOperands().get(1).getLiteral().getLongValue();
+ Assert.assertEquals(nowTs, 1577836800000L);
+ }
+
+ @Test
+ public void testCompilationInvokedNestedFunctions() {
+ String query = "SELECT a FROM foo where time_col > toDateTime(now(), 'yyyy-MM-dd z')";
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
+ Function greaterThan = pinotQuery.getFilterExpression().getFunctionCall();
+ String today = greaterThan.getOperands().get(1).getLiteral().getStringValue();
+ String expectedTodayStr = Instant.now().atZone(ZoneId.of("UTC")).format(DateTimeFormatter.ofPattern("yyyy-MM-dd z"));
+ Assert.assertEquals(today, expectedTodayStr);
}
}
\ No newline at end of file
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistry.java b/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistry.java
deleted file mode 100644
index 5a5ae5a..0000000
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistry.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.core.data.function;
-
-import com.google.common.base.Preconditions;
-import java.lang.reflect.Method;
-import java.util.HashMap;
-import java.util.Map;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * Registry for inbuilt Pinot functions
- */
-public class FunctionRegistry {
- private static final Logger LOGGER = LoggerFactory.getLogger(FunctionRegistry.class);
- private static final Map<String, FunctionInfo> _functionInfoMap = new HashMap<>();
-
- static {
- try {
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochSeconds", Long.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochMinutes", Long.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochHours", Long.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochDays", Long.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochSecondsRounded", Long.class, Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochMinutesRounded", Long.class, Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochHoursRounded", Long.class, Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochDaysRounded", Long.class, Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochSecondsBucket", Long.class, Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochMinutesBucket", Long.class, Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochHoursBucket", Long.class, Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochDaysBucket", Long.class, Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochSeconds", Long.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochMinutes", Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochHours", Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochDays", Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochSecondsBucket", Long.class, Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochMinutesBucket", Number.class, Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochHoursBucket", Number.class, Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochDaysBucket", Number.class, Number.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("toDateTime", Long.class, String.class));
- registerFunction(DateTimeFunctions.class.getDeclaredMethod("fromDateTime", String.class, String.class));
-
- registerFunction(JsonFunctions.class.getDeclaredMethod("toJsonMapStr", Map.class));
- } catch (NoSuchMethodException e) {
- LOGGER.error("Caught exception when registering function", e);
- throw new IllegalStateException(e);
- }
- }
-
- /**
- * Given a function name and a set of argument types, asserts that a corresponding function
- * was registered during construction and returns it
- */
- public static FunctionInfo getFunctionByNameWithApplicableArgumentTypes(String functionName,
- Class<?>[] argumentTypes) {
- Preconditions.checkArgument(_functionInfoMap.containsKey(functionName.toLowerCase()));
- FunctionInfo functionInfo = _functionInfoMap.get(functionName.toLowerCase());
- Preconditions.checkArgument(functionInfo.isApplicable(argumentTypes));
- return functionInfo;
- }
-
- static void registerFunction(Method method) {
- FunctionInfo functionInfo = new FunctionInfo(method, method.getDeclaringClass());
- _functionInfoMap.put(method.getName().toLowerCase(), functionInfo);
- }
-}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluator.java b/pinot-core/src/main/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluator.java
index 2cca5b0..e49d6c4 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluator.java
@@ -22,6 +22,9 @@ import com.google.common.base.Preconditions;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.pinot.common.function.FunctionInfo;
+import org.apache.pinot.common.function.FunctionInvoker;
+import org.apache.pinot.common.function.FunctionRegistry;
import org.apache.pinot.common.request.transform.TransformExpressionTree;
import org.apache.pinot.spi.data.readers.GenericRow;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/function/DateTimeFunctionEvaluatorTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/function/DateTimeFunctionEvaluatorTest.java
index 108ff71..9de5956 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/function/DateTimeFunctionEvaluatorTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/function/DateTimeFunctionEvaluatorTest.java
@@ -21,6 +21,7 @@ package org.apache.pinot.core.data.function;
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.List;
+import org.apache.pinot.common.function.DateTimeFunctions;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluatorTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluatorTest.java
index 8cdf72a..b89f1e2 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluatorTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluatorTest.java
@@ -19,6 +19,8 @@
package org.apache.pinot.core.data.function;
import com.google.common.collect.Lists;
+import org.apache.pinot.common.function.FunctionInfo;
+import org.apache.pinot.common.function.FunctionRegistry;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.joda.time.DateTime;
import org.joda.time.Days;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org