You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ro...@apache.org on 2022/07/12 18:58:42 UTC
[pinot] branch master updated: Support SET key=value syntax (#9017)
This is an automated email from the ASF dual-hosted git repository.
rongr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0b657cfcd4 Support SET key=value syntax (#9017)
0b657cfcd4 is described below
commit 0b657cfcd496fcec11ad371069982ab05b897df7
Author: Rong Rong <wa...@gmail.com>
AuthorDate: Tue Jul 12 11:58:36 2022 -0700
Support SET key=value syntax (#9017)
* support SET OPTIONS query syntax
* maintain deprecated OPTIONS keyword support, but clearly marked deprecation
* address diff comments
Co-authored-by: Rong Rong <ro...@startree.ai>
---
.../broker/api/resources/PinotClientRequest.java | 2 +-
.../requesthandler/BrokerRequestOptionsTest.java | 4 +-
.../broker/requesthandler/QueryValidationTest.java | 4 +-
.../src/main/codegen/includes/parserImpls.ftl | 3 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 125 ++++++++++++---------
.../pinot/sql/parsers/SqlNodeAndOptions.java | 19 +++-
.../pinot/sql/parsers/dml/InsertIntoFile.java | 3 +-
.../pinot/sql/parsers/CalciteSqlCompilerTest.java | 83 ++++++++++++--
.../pinot/sql/parsers/dml/InsertIntoFileTest.java | 12 +-
.../api/resources/PinotQueryResource.java | 4 +-
10 files changed, 179 insertions(+), 80 deletions(-)
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
index 491de31bc2..9e8037f329 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
@@ -154,7 +154,7 @@ public class PinotClientRequest {
} catch (Exception e) {
return new BrokerResponseNative(QueryException.getException(QueryException.SQL_PARSING_ERROR, e));
}
- PinotSqlType sqlType = CalciteSqlParser.extractSqlType(sqlNodeAndOptions.getSqlNode());
+ PinotSqlType sqlType = sqlNodeAndOptions.getSqlType();
if (onlyDql && sqlType != PinotSqlType.DQL) {
return new BrokerResponseNative(QueryException.getException(QueryException.SQL_PARSING_ERROR,
new UnsupportedOperationException("Unsupported SQL type - " + sqlType + ", GET API only supports DQL.")));
diff --git a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerRequestOptionsTest.java b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerRequestOptionsTest.java
index 18d01e088c..dc83218d08 100644
--- a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerRequestOptionsTest.java
+++ b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerRequestOptionsTest.java
@@ -104,7 +104,7 @@ public class BrokerRequestOptionsTest {
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption1"), "foo");
// Has queryOptions in query
- pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from testTable option(queryOption1=foo)");
+ pinotQuery = CalciteSqlParser.compileToPinotQuery("SET queryOption1='foo'; select * from testTable");
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query, jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
@@ -120,7 +120,7 @@ public class BrokerRequestOptionsTest {
// Has query options in both json payload and pinotQuery, pinotQuery takes priority
jsonRequest.put(Request.QUERY_OPTIONS, "queryOption1=bar;queryOption2=moo");
- pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from testTable option(queryOption1=foo)");
+ pinotQuery = CalciteSqlParser.compileToPinotQuery("SET queryOption1='foo'; select * from testTable;");
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query, jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 2);
diff --git a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/QueryValidationTest.java b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/QueryValidationTest.java
index bcc26c3a26..4772c0ddac 100644
--- a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/QueryValidationTest.java
+++ b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/QueryValidationTest.java
@@ -115,8 +115,8 @@ public class QueryValidationTest {
@Test
public void testReplicaGroupToQueryInvalidQuery() {
- PinotQuery pinotQuery =
- CalciteSqlParser.compileToPinotQuery("SELECT COUNT(*) FROM MY_TABLE OPTION(numReplicaGroupsToQuery=illegal)");
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(
+ "SET numReplicaGroupsToQuery='illegal'; SELECT COUNT(*) FROM MY_TABLE");
Assert.assertThrows(IllegalStateException.class, () -> BaseBrokerRequestHandler.validateRequest(pinotQuery, 10));
}
diff --git a/pinot-common/src/main/codegen/includes/parserImpls.ftl b/pinot-common/src/main/codegen/includes/parserImpls.ftl
index 57d3e2b8af..3b85f19aff 100644
--- a/pinot-common/src/main/codegen/includes/parserImpls.ftl
+++ b/pinot-common/src/main/codegen/includes/parserImpls.ftl
@@ -73,7 +73,8 @@ SqlInsertFromFile SqlInsertFromFile() :
}
}
-/* define the rest of the sql into SqlStmtList
+/**
+ * define the rest of the sql into SqlStmtList
*/
private void SqlStatementList(SqlNodeList list) :
{
diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
index 90a9863beb..3f3cc1fd7b 100644
--- a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
+++ b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
@@ -43,6 +43,7 @@ import org.apache.calcite.sql.SqlNumericLiteral;
import org.apache.calcite.sql.SqlOrderBy;
import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.SqlSelectKeyword;
+import org.apache.calcite.sql.SqlSetOption;
import org.apache.calcite.sql.fun.SqlBetweenOperator;
import org.apache.calcite.sql.fun.SqlCase;
import org.apache.calcite.sql.fun.SqlLikeOperator;
@@ -105,14 +106,14 @@ public class CalciteSqlParser {
}
public static SqlNodeAndOptions compileToSqlNodeAndOptions(String sql)
- throws Exception {
+ throws SqlCompilationException {
// Remove the comments from the query
sql = removeComments(sql);
// Remove the terminating semicolon from the query
sql = removeTerminatingSemicolon(sql);
- // Extract OPTION statements from sql as Calcite Parser doesn't parse it.
+ // extract and remove OPTIONS string
List<String> options = extractOptionsFromSql(sql);
if (!options.isEmpty()) {
sql = removeOptionsFromSql(sql);
@@ -120,51 +121,66 @@ public class CalciteSqlParser {
try (StringReader inStream = new StringReader(sql)) {
SqlParserImpl sqlParser = newSqlParser(inStream);
- return new SqlNodeAndOptions(sqlParser.parseSqlStmtEof(), options);
+ SqlNodeList sqlNodeList = sqlParser.SqlStmtsEof();
+ // Extract OPTION statements from sql.
+ SqlNodeAndOptions sqlNodeAndOptions = extractSqlNodeAndOptions(sqlNodeList);
+ // add legacy OPTIONS keyword-based options
+ if (options.size() > 0) {
+ sqlNodeAndOptions.setExtraOptions(extractOptionsMap(options));
+ }
+ return sqlNodeAndOptions;
} catch (Throwable e) {
throw new SqlCompilationException("Caught exception while parsing query: " + sql, e);
}
}
- public static PinotSqlType extractSqlType(SqlNode sqlNode) {
- switch (sqlNode.getKind()) {
- case OTHER_DDL:
- if (sqlNode instanceof SqlInsertFromFile) {
- return PinotSqlType.DML;
+ public static SqlNodeAndOptions extractSqlNodeAndOptions(SqlNodeList sqlNodeList) {
+ PinotSqlType sqlType = null;
+ SqlNode statementNode = null;
+ Map<String, String> options = new HashMap<>();
+ for (SqlNode sqlNode : sqlNodeList) {
+ if (sqlNode instanceof SqlInsertFromFile) {
+ // extract insert statement (execution statement)
+ if (sqlType == null) {
+ sqlType = PinotSqlType.DML;
+ statementNode = sqlNode;
+ } else {
+ throw new SqlCompilationException("SqlNode with executable statement already exist with type: " + sqlType);
}
- throw new SqlCompilationException("Unsupported SqlNode type - " + sqlNode.getKind());
- default:
- return PinotSqlType.DQL;
+ } else if (sqlNode instanceof SqlSetOption) {
+ // extract options, these are non-execution statements
+ List<SqlNode> operandList = ((SqlSetOption) sqlNode).getOperandList();
+ SqlIdentifier key = (SqlIdentifier) operandList.get(1);
+ SqlLiteral value = (SqlLiteral) operandList.get(2);
+ options.put(key.getSimple(), value.toValue());
+ } else {
+ // default extract query statement (execution statement)
+ if (sqlType == null) {
+ sqlType = PinotSqlType.DQL;
+ statementNode = sqlNode;
+ } else {
+ throw new SqlCompilationException("SqlNode with executable statement already exist with type: " + sqlType);
+ }
+ }
}
+ if (sqlType == null) {
+ throw new SqlCompilationException("SqlNode with executable statement not found!");
+ }
+ return new SqlNodeAndOptions(statementNode, sqlType, options);
}
public static PinotQuery compileToPinotQuery(String sql)
throws SqlCompilationException {
- // Remove the comments from the query
- sql = removeComments(sql);
-
- // Remove the terminating semicolon from the query
- sql = removeTerminatingSemicolon(sql);
-
- // Extract OPTION statements from sql as Calcite Parser doesn't parse it.
- List<String> options = extractOptionsFromSql(sql);
- if (!options.isEmpty()) {
- sql = removeOptionsFromSql(sql);
- }
-
- SqlNode sqlNode;
- try (StringReader inStream = new StringReader(sql)) {
- SqlParserImpl sqlParser = newSqlParser(inStream);
- sqlNode = sqlParser.parseSqlStmtEof();
- } catch (Throwable e) {
- throw new SqlCompilationException("Caught exception while parsing query: " + sql, e);
- }
+ SqlNodeAndOptions sqlNodeAndOptions = compileToSqlNodeAndOptions(sql);
// Compile Sql without OPTION statements.
- PinotQuery pinotQuery = compileSqlNodeToPinotQuery(sqlNode);
+ PinotQuery pinotQuery = compileSqlNodeToPinotQuery(sqlNodeAndOptions.getSqlNode());
// Set Option statements to PinotQuery.
- setOptions(pinotQuery, options);
+ Map<String, String> options = sqlNodeAndOptions.getOptions();
+ if (!options.isEmpty()) {
+ pinotQuery.setQueryOptions(options);
+ }
return pinotQuery;
}
@@ -355,27 +371,6 @@ public class CalciteSqlParser {
return sqlParser;
}
- public static Map<String, String> extractOptionsMap(List<String> optionsStatements) {
- Map<String, String> options = new HashMap<>();
- for (String optionsStatement : optionsStatements) {
- for (String option : optionsStatement.split(",")) {
- final String[] splits = option.split("=");
- if (splits.length != 2) {
- throw new SqlCompilationException("OPTION statement requires two parts separated by '='");
- }
- options.put(splits[0].trim(), splits[1].trim());
- }
- }
- return options;
- }
-
- private static void setOptions(PinotQuery pinotQuery, List<String> optionsStatements) {
- if (optionsStatements.isEmpty()) {
- return;
- }
- pinotQuery.setQueryOptions(extractOptionsMap(optionsStatements));
- }
-
public static PinotQuery compileSqlNodeToPinotQuery(SqlNode sqlNode) {
PinotQuery pinotQuery = new PinotQuery();
if (sqlNode instanceof SqlExplain) {
@@ -460,6 +455,7 @@ public class CalciteSqlParser {
validate(pinotQuery);
}
+ @Deprecated
private static List<String> extractOptionsFromSql(String sql) {
List<String> results = new ArrayList<>();
Matcher matcher = OPTIONS_REGEX_PATTEN.matcher(sql);
@@ -469,11 +465,34 @@ public class CalciteSqlParser {
return results;
}
+ @Deprecated
private static String removeOptionsFromSql(String sql) {
Matcher matcher = OPTIONS_REGEX_PATTEN.matcher(sql);
return matcher.replaceAll("");
}
+ @Deprecated
+ private static Map<String, String> extractOptionsMap(List<String> optionsStatements) {
+ Map<String, String> options = new HashMap<>();
+ for (String optionsStatement : optionsStatements) {
+ for (String option : optionsStatement.split(",")) {
+ final String[] splits = option.split("=");
+ if (splits.length != 2) {
+ throw new SqlCompilationException("OPTION statement requires two parts separated by '='");
+ }
+ options.put(splits[0].trim(), splits[1].trim());
+ }
+ }
+ return options;
+ }
+
+ private static void setOptions(PinotQuery pinotQuery, List<String> optionsStatements) {
+ if (optionsStatements.isEmpty()) {
+ return;
+ }
+ pinotQuery.setQueryOptions(extractOptionsMap(optionsStatements));
+ }
+
/**
* Removes comments from the query.
* NOTE: Comment indicator within single quotes (literal) and double quotes (identifier) are ignored.
diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/SqlNodeAndOptions.java b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/SqlNodeAndOptions.java
index 9e81da7378..92b7183be0 100644
--- a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/SqlNodeAndOptions.java
+++ b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/SqlNodeAndOptions.java
@@ -18,16 +18,19 @@
*/
package org.apache.pinot.sql.parsers;
-import java.util.List;
+import java.util.Map;
import org.apache.calcite.sql.SqlNode;
public class SqlNodeAndOptions {
private final SqlNode _sqlNode;
- private final List<String> _options;
+ private final PinotSqlType _sqlType;
+ // TODO: support option literals other than STRING
+ private final Map<String, String> _options;
- public SqlNodeAndOptions(SqlNode sqlNode, List<String> options) {
+ public SqlNodeAndOptions(SqlNode sqlNode, PinotSqlType sqlType, Map<String, String> options) {
_sqlNode = sqlNode;
+ _sqlType = sqlType;
_options = options;
}
@@ -35,7 +38,15 @@ public class SqlNodeAndOptions {
return _sqlNode;
}
- public List<String> getOptions() {
+ public Map<String, String> getOptions() {
return _options;
}
+
+ public PinotSqlType getSqlType() {
+ return _sqlType;
+ }
+
+ public void setExtraOptions(Map<String, String> extractOptionsMap) {
+ _options.putAll(extractOptionsMap);
+ }
}
diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/dml/InsertIntoFile.java b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/dml/InsertIntoFile.java
index aac4199496..6106a2e028 100644
--- a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/dml/InsertIntoFile.java
+++ b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/dml/InsertIntoFile.java
@@ -26,7 +26,6 @@ import org.apache.calcite.sql.SqlNodeList;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.spi.config.task.AdhocTaskConfig;
-import org.apache.pinot.sql.parsers.CalciteSqlParser;
import org.apache.pinot.sql.parsers.SqlNodeAndOptions;
import org.apache.pinot.sql.parsers.parser.SqlInsertFromFile;
@@ -69,7 +68,7 @@ public class InsertIntoFile implements DataManipulationStatement {
String tableName = operandList.get(0) != null ? StringUtils.joinWith(",", operandList.get(0), operandList.get(1))
: operandList.get(1).toString();
// Set Options
- Map<String, String> optionsMap = CalciteSqlParser.extractOptionsMap(sqlNodeAndOptions.getOptions());
+ Map<String, String> optionsMap = sqlNodeAndOptions.getOptions();
List<String> inputDirList = new ArrayList<>();
((SqlNodeList) operandList.get(2)).getList()
.forEach(sqlNode1 -> inputDirList.add(sqlNode1.toString().replace("'", "")));
diff --git a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
index 8cbd47fa32..a022d1bb8f 100644
--- a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
@@ -24,7 +24,7 @@ import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.concurrent.TimeUnit;
-import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlNumericLiteral;
import org.apache.pinot.common.request.Expression;
import org.apache.pinot.common.request.ExpressionType;
@@ -692,6 +692,8 @@ public class CalciteSqlCompilerTest {
}
@Test
+ @Deprecated
+ // TODO: to be removed once OPTIONS REGEX match is deprecated
public void testQueryOptions() {
PinotQuery pinotQuery =
CalciteSqlParser.compileToPinotQuery("select * from vegetables where name <> 'Brussels sprouts'");
@@ -729,6 +731,69 @@ public class CalciteSqlCompilerTest {
}
}
+ @Test
+ public void testQuerySetOptions() {
+ PinotQuery pinotQuery =
+ CalciteSqlParser.compileToPinotQuery("select * from vegetables where name <> 'Brussels sprouts'");
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 0);
+ Assert.assertNull(pinotQuery.getQueryOptions());
+
+ pinotQuery = CalciteSqlParser.compileToPinotQuery(
+ "SET delicious='yes'; select * from vegetables where name <> 'Brussels sprouts'");
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
+ Assert.assertTrue(pinotQuery.getQueryOptions().containsKey("delicious"));
+ Assert.assertEquals(pinotQuery.getQueryOptions().get("delicious"), "yes");
+
+ pinotQuery = CalciteSqlParser.compileToPinotQuery("SET delicious='yes'; SET foo='1234'; SET bar='''potato''';"
+ + "select * from vegetables where name <> 'Brussels sprouts' ");
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 3);
+ Assert.assertTrue(pinotQuery.getQueryOptions().containsKey("delicious"));
+ Assert.assertEquals(pinotQuery.getQueryOptions().get("delicious"), "yes");
+ Assert.assertEquals(pinotQuery.getQueryOptions().get("foo"), "1234");
+ Assert.assertEquals(pinotQuery.getQueryOptions().get("bar"), "'potato'");
+
+ pinotQuery = CalciteSqlParser.compileToPinotQuery("SET delicious='yes'; SET foo='1234'; "
+ + "SET bar='''potato'''; select * from vegetables where name <> 'Brussels sprouts' ");
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 3);
+ Assert.assertTrue(pinotQuery.getQueryOptions().containsKey("delicious"));
+ Assert.assertEquals(pinotQuery.getQueryOptions().get("delicious"), "yes");
+ Assert.assertEquals(pinotQuery.getQueryOptions().get("foo"), "1234");
+ Assert.assertEquals(pinotQuery.getQueryOptions().get("bar"), "'potato'");
+
+ pinotQuery = CalciteSqlParser.compileToPinotQuery("SET delicious='yes'; SET foo='1234'; "
+ + "select * from vegetables where name <> 'Brussels sprouts'; SET bar='''potato'''; ");
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 3);
+ Assert.assertTrue(pinotQuery.getQueryOptions().containsKey("delicious"));
+ Assert.assertEquals(pinotQuery.getQueryOptions().get("delicious"), "yes");
+ Assert.assertEquals(pinotQuery.getQueryOptions().get("foo"), "1234");
+ Assert.assertEquals(pinotQuery.getQueryOptions().get("bar"), "'potato'");
+
+ // test invalid options
+ try {
+ CalciteSqlParser.compileToPinotQuery("select * from vegetables SET delicious='yes', foo='1234' "
+ + "where name <> 'Brussels sprouts'");
+ Assert.fail("SQL should not be compiled");
+ } catch (SqlCompilationException sce) {
+ // expected.
+ }
+
+ try {
+ CalciteSqlParser.compileToPinotQuery("select * from vegetables where name <> 'Brussels sprouts'; "
+ + "SET (delicious='yes', foo=1234)");
+ Assert.fail("SQL should not be compiled");
+ } catch (SqlCompilationException sce) {
+ // expected.
+ }
+
+ try {
+ CalciteSqlParser.compileToPinotQuery("select * from vegetables where name <> 'Brussels sprouts'; "
+ + "SET (delicious='yes', foo=1234); select * from meat");
+ Assert.fail("SQL should not be compiled");
+ } catch (SqlCompilationException sce) {
+ // expected.
+ }
+ }
+
@Test
public void testRemoveComments() {
testRemoveComments("select * from myTable", "select * from myTable");
@@ -2422,12 +2487,14 @@ public class CalciteSqlCompilerTest {
Assert.assertEquals(pinotQuery.getGroupByList().get(0).getIdentifier().getName(), "col1");
// Check for Option SQL Query
+ // TODO: change to SET syntax
sql = "SELECT col1, count(*) FROM foo group by col1 option(skipUpsert=true);";
pinotQuery = CalciteSqlParser.compileToPinotQuery(sql);
Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
Assert.assertTrue(pinotQuery.getQueryOptions().containsKey("skipUpsert"));
// Check for the query where the literal has semicolon
+ // TODO: change to SET syntax
sql = "select col1, count(*) from foo where col1 = 'x;y' GROUP BY col1 option(skipUpsert=true);";
pinotQuery = CalciteSqlParser.compileToPinotQuery(sql);
Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
@@ -2471,18 +2538,20 @@ public class CalciteSqlCompilerTest {
@Test
public void testParserExtensionImpl() {
String customSql = "INSERT INTO db.tbl FROM FILE 'file:///tmp/file1', FILE 'file:///tmp/file2'";
- SqlNode sqlNode = testSqlWithCustomSqlParser(customSql);
- Assert.assertTrue(sqlNode instanceof SqlInsertFromFile);
- Assert.assertEquals(CalciteSqlParser.extractSqlType(sqlNode), PinotSqlType.DML);
+ SqlNodeAndOptions sqlNodeAndOptions = testSqlWithCustomSqlParser(customSql);
+ Assert.assertTrue(sqlNodeAndOptions.getSqlNode() instanceof SqlInsertFromFile);
+ Assert.assertEquals(sqlNodeAndOptions.getSqlType(), PinotSqlType.DML);
}
- private static SqlNode testSqlWithCustomSqlParser(String sqlString) {
+ private static SqlNodeAndOptions testSqlWithCustomSqlParser(String sqlString) {
try (StringReader inStream = new StringReader(sqlString)) {
SqlParserImpl sqlParser = CalciteSqlParser.newSqlParser(inStream);
- return sqlParser.parseSqlStmtEof();
+ SqlNodeList sqlNodeList = sqlParser.SqlStmtsEof();
+ // Extract OPTION statements from sql.
+ return CalciteSqlParser.extractSqlNodeAndOptions(sqlNodeList);
} catch (Exception e) {
Assert.fail("test custom sql parser failed", e);
+ return null;
}
- return null;
}
}
diff --git a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/dml/InsertIntoFileTest.java b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/dml/InsertIntoFileTest.java
index 2092f1b18f..eaf7368f73 100644
--- a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/dml/InsertIntoFileTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/dml/InsertIntoFileTest.java
@@ -31,12 +31,12 @@ public class InsertIntoFileTest {
public void testInsertIntoStatementParser()
throws Exception {
String insertIntoSql = "INSERT INTO \"baseballStats\"\n"
- + "FROM FILE 's3://my-bucket/path/to/data/'\n"
- + "OPTION(taskName=myTask-1,"
- + "input.fs.className=org.apache.pinot.plugin.filesystem.S3PinotFS,"
- + "input.fs.prop.accessKey=my-access-key,"
- + "input.fs.prop.secretKey=my-secret-key,"
- + "input.fs.prop.region=us-west-2)";
+ + "FROM FILE 's3://my-bucket/path/to/data/';\n"
+ + "SET taskName = 'myTask-1';\n"
+ + "SET \"input.fs.className\" = 'org.apache.pinot.plugin.filesystem.S3PinotFS';\n"
+ + "SET \"input.fs.prop.accessKey\" = 'my-access-key';\n"
+ + "SET \"input.fs.prop.secretKey\" = 'my-secret-key';\n"
+ + "SET \"input.fs.prop.region\" = 'us-west-2';";
InsertIntoFile insertIntoFile = InsertIntoFile.parse(CalciteSqlParser.compileToSqlNodeAndOptions(insertIntoSql));
Assert.assertEquals(insertIntoFile.getTable(), "baseballStats");
Assert.assertEquals(insertIntoFile.getExecutionType(), DataManipulationStatement.ExecutionType.MINION);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
index f79402bac1..a21279c549 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
@@ -108,7 +108,7 @@ public class PinotQueryResource {
String queryOptions)
throws Exception {
SqlNodeAndOptions sqlNodeAndOptions = CalciteSqlParser.compileToSqlNodeAndOptions(sqlQuery);
- PinotSqlType sqlType = CalciteSqlParser.extractSqlType(sqlNodeAndOptions.getSqlNode());
+ PinotSqlType sqlType = sqlNodeAndOptions.getSqlType();
switch (sqlType) {
case DQL:
return getQueryResponse(sqlQuery, sqlNodeAndOptions.getSqlNode(), traceEnabled, queryOptions, httpHeaders);
@@ -130,7 +130,7 @@ public class PinotQueryResource {
try {
LOGGER.debug("Trace: {}, Running query: {}", traceEnabled, sqlQuery);
SqlNodeAndOptions sqlNodeAndOptions = CalciteSqlParser.compileToSqlNodeAndOptions(sqlQuery);
- PinotSqlType sqlType = CalciteSqlParser.extractSqlType(sqlNodeAndOptions.getSqlNode());
+ PinotSqlType sqlType = sqlNodeAndOptions.getSqlType();
if (sqlType == PinotSqlType.DQL) {
return getQueryResponse(sqlQuery, sqlNodeAndOptions.getSqlNode(), traceEnabled, queryOptions, httpHeaders);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org