[incubator-pinot] 01/01: Adding support to execute functions during query compilation

[ki...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kishoreg pushed a commit to branch time-functions-in-query
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git

commit b91b4f33292c594307fe5e7087beae7f13dc0531
Author: kishoreg <g....@gmail.com>
AuthorDate: Mon May 18 00:25:02 2020 -0700

    Adding support to execute functions during query compilation
---
 .../function/FunctionDefinitionRegistry.java       |  4 ++
 .../pinot/common/function/ScalarFunctionType.java  | 69 ++++++++++++++++++++++
 .../apache/pinot/sql/parsers/CalciteSqlParser.java | 45 +++++++++++++-
 3 files changed, 117 insertions(+), 1 deletion(-)

diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionDefinitionRegistry.java b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionDefinitionRegistry.java
index 8e70a20..25381c3 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionDefinitionRegistry.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionDefinitionRegistry.java
@@ -40,4 +40,8 @@ public class FunctionDefinitionRegistry {
       return false;
     }
   }
+
+  public static boolean isScalarFunc(String functionName) {
+    return ScalarFunctionType.isScalarFunctionType(functionName);
+  }
 }
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/ScalarFunctionType.java b/pinot-common/src/main/java/org/apache/pinot/common/function/ScalarFunctionType.java
new file mode 100644
index 0000000..c2c88a7
--- /dev/null
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/ScalarFunctionType.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.function;
+
+import java.util.HashMap;
+import java.util.HashSet;
+
+
+public enum ScalarFunctionType {
+
+  NOW("now"), FORMAT_DATETIME("format_datetime");
+
+  private final String _name;
+
+  static HashMap<String, ScalarFunctionType> _scalarFunctions = new HashMap<>();
+
+  static {
+    ScalarFunctionType[] values = ScalarFunctionType.values();
+    for (ScalarFunctionType value : values) {
+      String upperCaseFunctionName = value.getName().toUpperCase();
+      _scalarFunctions.put(upperCaseFunctionName, value);
+      _scalarFunctions.put(upperCaseFunctionName.replace("_", ""), value);
+    }
+  }
+
+  ScalarFunctionType(String name) {
+    _name = name;
+  }
+
+  /**
+   * Returns the corresponding transform function type for the given function name.
+   */
+  public static ScalarFunctionType getScalarFunctionType(String functionName) {
+    String upperCaseFunctionName = functionName.toUpperCase();
+    try {
+      return ScalarFunctionType.valueOf(upperCaseFunctionName);
+    } catch (Exception e) {
+      // Support function name of both datetime_format and datetime_format
+      if (upperCaseFunctionName.contains("_")) {
+        return getScalarFunctionType(upperCaseFunctionName.replace("_", ""));
+      }
+      throw new IllegalArgumentException("Invalid scalar function name: " + functionName);
+    }
+  }
+
+  public static boolean isScalarFunctionType(String functionName) {
+    return _scalarFunctions.containsKey(functionName.toUpperCase());
+  }
+
+  public String getName() {
+    return _name;
+  }
+}
diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
index 5abffc3..f63b703 100644
--- a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
+++ b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.sql.parsers;
 
+import com.google.common.base.Preconditions;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -45,6 +46,7 @@ import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl;
 import org.apache.calcite.sql.validate.SqlConformanceEnum;
 import org.apache.pinot.common.function.AggregationFunctionType;
 import org.apache.pinot.common.function.FunctionDefinitionRegistry;
+import org.apache.pinot.common.function.ScalarFunctionType;
 import org.apache.pinot.common.request.DataSource;
 import org.apache.pinot.common.request.Expression;
 import org.apache.pinot.common.request.ExpressionType;
@@ -52,6 +54,7 @@ import org.apache.pinot.common.request.Function;
 import org.apache.pinot.common.request.Identifier;
 import org.apache.pinot.common.request.PinotQuery;
 import org.apache.pinot.common.utils.request.RequestUtils;
+import org.joda.time.format.DateTimeFormat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -603,7 +606,7 @@ public class CalciteSqlParser {
             .equalsIgnoreCase(AggregationFunctionType.DISTINCT.getName())) {
           funcName = AggregationFunctionType.DISTINCTCOUNT.getName();
         }
-        final Expression funcExpr = RequestUtils.getFunctionExpression(funcName);
+        Expression funcExpr = RequestUtils.getFunctionExpression(funcName);
         for (SqlNode child : funcSqlNode.getOperands()) {
           if (child instanceof SqlNodeList) {
             final Iterator<SqlNode> iterator = ((SqlNodeList) child).iterator();
@@ -615,7 +618,47 @@ public class CalciteSqlParser {
             funcExpr.getFunctionCall().addToOperands(toExpression(child));
           }
         }
+
+        if (FunctionDefinitionRegistry.isScalarFunc(funcName) && isCompileTimeEvaluationPossible(funcExpr)) {
+
+          ScalarFunctionType scalarFunctionType = ScalarFunctionType.getScalarFunctionType(funcName);
+          switch (scalarFunctionType) {
+            case NOW:
+              funcExpr = RequestUtils.getLiteralExpression(System.currentTimeMillis());
+              break;
+            case FORMAT_DATETIME:
+              //DATETIME_FORMAT ('2020-01-01', 'yyyy-MM-dd')
+              String input = funcExpr.getFunctionCall().getOperands().get(0).getLiteral().getStringValue();
+              String format = funcExpr.getFunctionCall().getOperands().get(1).getLiteral().getStringValue();
+              long output = DateTimeFormat.forPattern(format).parseMillis(input);
+              funcExpr = RequestUtils.getLiteralExpression(output);
+              break;
+            default:
+              //no change, let the expression be handled during execution phase
+          }
+        }
+
         return funcExpr;
     }
   }
+
+  /**
+   * Utility method to check if the function can be evaluated during the query compilation phae
+   * @param funcExpr
+   * @return true if all arguments are literals
+   */
+  private static boolean isCompileTimeEvaluationPossible(Expression funcExpr) {
+
+    boolean compileTimeEvaluationPossible = true;
+    Function functionCall = funcExpr.getFunctionCall();
+    if(functionCall.getOperandsSize() > 0) {
+      for (Expression expression : functionCall.getOperands()) {
+        if (expression.getType() != ExpressionType.LITERAL) {
+          compileTimeEvaluationPossible = false;
+          break;
+        }
+      }
+    }
+    return compileTimeEvaluationPossible;
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org