You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ro...@apache.org on 2022/10/27 04:18:17 UTC
[pinot] branch master updated: [multistage] create pinot override SqlOpTable (#9643)
This is an automated email from the ASF dual-hosted git repository.
rongr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 585e0448d4 [multistage] create pinot override SqlOpTable (#9643)
585e0448d4 is described below
commit 585e0448d47939e6984676e44db808f6e531d671
Author: Rong Rong <ro...@apache.org>
AuthorDate: Wed Oct 26 21:18:11 2022 -0700
[multistage] create pinot override SqlOpTable (#9643)
* Create pinot override SqlOpTable
- use to fix COALESCE parsing issue
- also benefit from future non-function OP registration
* address diff comments
Co-authored-by: Rong Rong <ro...@startree.ai>
---
.../apache/calcite/sql/fun/PinotOperatorTable.java | 103 +++++++++++++++++++++
.../calcite/sql/fun/PinotSqlCoalesceFunction.java | 35 +++++++
.../org/apache/pinot/query/QueryEnvironment.java | 6 +-
.../java/org/apache/pinot/query/QueryTestSet.java | 8 +-
4 files changed, 149 insertions(+), 3 deletions(-)
diff --git a/pinot-query-planner/src/main/java/org/apache/calcite/sql/fun/PinotOperatorTable.java b/pinot-query-planner/src/main/java/org/apache/calcite/sql/fun/PinotOperatorTable.java
new file mode 100644
index 0000000000..ff37a2fbff
--- /dev/null
+++ b/pinot-query-planner/src/main/java/org/apache/calcite/sql/fun/PinotOperatorTable.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.calcite.sql.fun;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.validate.SqlNameMatchers;
+import org.apache.calcite.util.Util;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+
+
+/**
+ * {@link PinotOperatorTable} defines the {@link SqlOperator} overrides on top of the {@link SqlStdOperatorTable}.
+ *
+ * <p>The main purpose of this Pinot specific SQL operator table is to
+ * <ul>
+ * <li>Ensure that any specific SQL validation rules can apply with Pinot override entirely over Calcite's.</li>
+ * <li>Ability to create customer operators that are not function and cannot use
+ * {@link org.apache.calcite.prepare.Prepare.CatalogReader} to override</li>
+ * <li>Still maintain minimum customization and benefit from Calcite's original operator table setting.</li>
+ * </ul>
+ */
+public class PinotOperatorTable extends SqlStdOperatorTable {
+
+ private static @MonotonicNonNull PinotOperatorTable _instance;
+
+ public static final SqlFunction COALESCE = new PinotSqlCoalesceFunction();
+
+ // TODO: clean up lazy init by using Suppliers.memorized(this::computeInstance) and make getter wrapped around
+ // supplier instance. this should replace all lazy init static objects in the codebase
+ public static synchronized PinotOperatorTable instance() {
+ if (_instance == null) {
+ // Creates and initializes the standard operator table.
+ // Uses two-phase construction, because we can't initialize the
+ // table until the constructor of the sub-class has completed.
+ _instance = new PinotOperatorTable();
+ _instance.initNoDuplicate();
+ }
+ return _instance;
+ }
+
+ /**
+ * Initialize without duplicate, e.g. when 2 duplicate operator is linked with the same op
+ * {@link org.apache.calcite.sql.SqlKind} it causes problem.
+ *
+ * <p>This is a direct copy of the {@link org.apache.calcite.sql.util.ReflectiveSqlOperatorTable} and can be hard to
+ * debug, suggest changing to a non-dynamic registration. Dynamic function support should happen via catalog.
+ */
+ public final void initNoDuplicate() {
+ // Use reflection to register the expressions stored in public fields.
+ for (Field field : getClass().getFields()) {
+ try {
+ if (SqlFunction.class.isAssignableFrom(field.getType())) {
+ SqlFunction op = (SqlFunction) field.get(this);
+ if (op != null && notRegistered(op)) {
+ register(op);
+ }
+ } else if (
+ SqlOperator.class.isAssignableFrom(field.getType())) {
+ SqlOperator op = (SqlOperator) field.get(this);
+ if (op != null && notRegistered(op)) {
+ register(op);
+ }
+ }
+ } catch (IllegalArgumentException | IllegalAccessException e) {
+ throw Util.throwAsRuntime(Util.causeOrSelf(e));
+ }
+ }
+ }
+
+ private boolean notRegistered(SqlFunction op) {
+ List<SqlOperator> operatorList = new ArrayList<>();
+ lookupOperatorOverloads(op.getNameAsId(), op.getFunctionType(), op.getSyntax(), operatorList,
+ SqlNameMatchers.withCaseSensitive(false));
+ return operatorList.size() == 0;
+ }
+
+ private boolean notRegistered(SqlOperator op) {
+ List<SqlOperator> operatorList = new ArrayList<>();
+ lookupOperatorOverloads(op.getNameAsId(), null, op.getSyntax(), operatorList,
+ SqlNameMatchers.withCaseSensitive(false));
+ return operatorList.size() == 0;
+ }
+}
diff --git a/pinot-query-planner/src/main/java/org/apache/calcite/sql/fun/PinotSqlCoalesceFunction.java b/pinot-query-planner/src/main/java/org/apache/calcite/sql/fun/PinotSqlCoalesceFunction.java
new file mode 100644
index 0000000000..92ef85857f
--- /dev/null
+++ b/pinot-query-planner/src/main/java/org/apache/calcite/sql/fun/PinotSqlCoalesceFunction.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.calcite.sql.fun;
+
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.validate.SqlValidator;
+
+
+/**
+ * Pinot supports native COALESCE function, thus no need to create CASE WHEN conversion.
+ */
+public class PinotSqlCoalesceFunction extends SqlCoalesceFunction {
+
+ @Override
+ public SqlNode rewriteCall(SqlValidator validator, SqlCall call) {
+ return call;
+ }
+}
diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/QueryEnvironment.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/QueryEnvironment.java
index 0a661ac378..c1d111fe5b 100644
--- a/pinot-query-planner/src/main/java/org/apache/pinot/query/QueryEnvironment.java
+++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/QueryEnvironment.java
@@ -43,7 +43,7 @@ import org.apache.calcite.sql.SqlExplainFormat;
import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.fun.PinotOperatorTable;
import org.apache.calcite.sql.util.ChainedSqlOperatorTable;
import org.apache.calcite.sql2rel.SqlToRelConverter;
import org.apache.calcite.sql2rel.StandardConvertletTable;
@@ -91,7 +91,9 @@ public class QueryEnvironment {
new CalciteConnectionConfigImpl(catalogReaderConfigProperties));
_config = Frameworks.newConfigBuilder().traitDefs()
- .operatorTable(new ChainedSqlOperatorTable(Arrays.asList(SqlStdOperatorTable.instance(), _catalogReader)))
+ .operatorTable(new ChainedSqlOperatorTable(Arrays.asList(
+ PinotOperatorTable.instance(),
+ _catalogReader)))
.defaultSchema(_rootSchema.plus())
.sqlToRelConverterConfig(SqlToRelConverter.config()
.withHintStrategyTable(getHintStrategyTable())
diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/QueryTestSet.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/QueryTestSet.java
index a4800f4834..f8f90a7f2c 100644
--- a/pinot-query-runtime/src/test/java/org/apache/pinot/query/QueryTestSet.java
+++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/QueryTestSet.java
@@ -207,10 +207,16 @@ public class QueryTestSet {
new Object[]{"SELECT SUM(CAST(col3 AS INTEGER)) FROM a HAVING MIN(col3) BETWEEN 1 AND 0"},
new Object[]{"SELECT col1, COUNT(col3) FROM a GROUP BY col1 HAVING SUM(col3) > 40 AND SUM(col3) < 30"},
- // Test functions
+ // Test SQL functions
+ // TODO split these SQL functions into separate test files to share between planner and runtime
+ // LIKE function
new Object[]{"SELECT col1 FROM a WHERE col2 LIKE '%o%'"},
new Object[]{"SELECT a.col1, b.col1 FROM a JOIN b ON a.col3 = b.col3 WHERE a.col2 LIKE b.col1"},
new Object[]{"SELECT a.col1 LIKE b.col1 FROM a JOIN b ON a.col3 = b.col3"},
+
+ // COALESCE function
+ new Object[]{"SELECT a.col1, COALESCE(b.col3, 0) FROM a LEFT JOIN b ON a.col1 = b.col2"},
+ new Object[]{"SELECT a.col1, COALESCE(a.col3, 0) FROM a WHERE COALESCE(a.col2, 'bar') = 'bar'"},
};
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org