You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ro...@apache.org on 2022/07/26 20:22:25 UTC
[pinot] branch master updated: [UI] Add controller UI to use multi-stage engine (#9072)
This is an automated email from the ASF dual-hosted git repository.
rongr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0abd8f87ab [UI] Add controller UI to use multi-stage engine (#9072)
0abd8f87ab is described below
commit 0abd8f87abdffe304121982597e329213688ab8c
Author: Rong Rong <wa...@gmail.com>
AuthorDate: Tue Jul 26 13:22:19 2022 -0700
[UI] Add controller UI to use multi-stage engine (#9072)
* adding controller endpoint to poke multi-stage engine
* adding controller API to accept query option for using multistage engine
* changing the controller UI to show new Engine
* remove pql query options
* make non-supported exception more clear
Co-authored-by: Rong Rong <ro...@startree.ai>
---
.../broker/api/resources/PinotClientRequest.java | 15 +---
.../requesthandler/BaseBrokerRequestHandler.java | 6 +-
.../BrokerRequestHandlerDelegate.java | 10 ++-
.../requesthandler/BrokerRequestOptionsTest.java | 20 ++---
.../api/resources/PinotQueryResource.java | 92 +++++++++++++++-------
.../src/main/resources/app/pages/Query.tsx | 44 ++++++++---
.../main/resources/app/utils/PinotMethodUtils.ts | 43 +---------
.../tests/MultiStageEngineIntegrationTest.java | 3 +-
.../pinot/tools/MultistageEngineQuickStart.java | 10 +--
9 files changed, 130 insertions(+), 113 deletions(-)
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
index 9e8037f329..9bc1b466c0 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
@@ -100,8 +100,6 @@ public class PinotClientRequest {
try {
ObjectNode requestJson = JsonUtils.newObjectNode();
requestJson.put(Request.SQL, query);
- String queryOptions = constructSqlQueryOptions();
- requestJson.put(Request.QUERY_OPTIONS, queryOptions);
if (traceEnabled != null) {
requestJson.put(Request.TRACE, traceEnabled);
}
@@ -133,10 +131,8 @@ public class PinotClientRequest {
if (!requestJson.has(Request.SQL)) {
throw new IllegalStateException("Payload is missing the query string field 'sql'");
}
- String queryOptions = constructSqlQueryOptions();
- // the only query options as of now are sql related. do not allow any custom query options in sql endpoint
- ObjectNode sqlRequestJson = ((ObjectNode) requestJson).put(Request.QUERY_OPTIONS, queryOptions);
- BrokerResponse brokerResponse = executeSqlQuery(sqlRequestJson, makeHttpIdentity(requestContext), false);
+ BrokerResponse brokerResponse = executeSqlQuery((ObjectNode) requestJson, makeHttpIdentity(requestContext),
+ false);
asyncResponse.resume(brokerResponse.toJsonString());
} catch (Exception e) {
LOGGER.error("Caught exception while processing POST request", e);
@@ -174,13 +170,6 @@ public class PinotClientRequest {
new UnsupportedOperationException("Unsupported SQL type - " + sqlType)));
}
}
-
- // TODO: Remove the SQL query options after releasing 0.11.0
- private String constructSqlQueryOptions() {
- return Request.QueryOptionKey.GROUP_BY_MODE + "=" + Request.SQL + ";" + Request.QueryOptionKey.RESPONSE_FORMAT + "="
- + Request.SQL;
- }
-
private static HttpRequesterIdentity makeHttpIdentity(org.glassfish.grizzly.http.server.Request context) {
Multimap<String, String> headers = ArrayListMultimap.create();
context.getHeaderNames().forEach(key -> context.getHeaders(key).forEach(value -> headers.put(key, value)));
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
index 8320f17681..3f2bd37322 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
@@ -1420,7 +1420,7 @@ public abstract class BaseBrokerRequestHandler implements BrokerRequestHandler {
throw new BadQueryRequestException("Unknown columnName '" + columnName + "' found in the query");
}
- private static Map<String, String> getOptionsFromJson(JsonNode request, String optionsKey) {
+ public static Map<String, String> getOptionsFromJson(JsonNode request, String optionsKey) {
return Splitter.on(';').omitEmptyStrings().trimResults().withKeyValueSeparator('=')
.split(request.get(optionsKey).asText());
}
@@ -1477,6 +1477,10 @@ public abstract class BaseBrokerRequestHandler implements BrokerRequestHandler {
if (!queryOptions.isEmpty()) {
LOGGER.debug("Query options are set to: {} for request {}: {}", queryOptions, requestId, query);
}
+ // TODO: Remove the SQL query options after releasing 0.11.0
+ // The query engine will break if these 2 options are missing during version upgrade.
+ queryOptions.put(Broker.Request.QueryOptionKey.GROUP_BY_MODE, Broker.Request.SQL);
+ queryOptions.put(Broker.Request.QueryOptionKey.RESPONSE_FORMAT, Broker.Request.SQL);
}
/**
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BrokerRequestHandlerDelegate.java b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BrokerRequestHandlerDelegate.java
index a39ad1d265..046e60c04f 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BrokerRequestHandlerDelegate.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BrokerRequestHandlerDelegate.java
@@ -19,6 +19,7 @@
package org.apache.pinot.broker.requesthandler;
import com.fasterxml.jackson.databind.JsonNode;
+import java.util.Map;
import javax.annotation.Nullable;
import org.apache.pinot.broker.api.RequesterIdentity;
import org.apache.pinot.common.response.BrokerResponse;
@@ -74,9 +75,12 @@ public class BrokerRequestHandlerDelegate implements BrokerRequestHandler {
RequestContext requestContext)
throws Exception {
if (_isMultiStageQueryEngineEnabled && _multiStageWorkerRequestHandler != null) {
- JsonNode node = request.get(CommonConstants.Broker.Request.QueryOptionKey.USE_MULTISTAGE_ENGINE);
- if (node != null && node.asBoolean()) {
- return _multiStageWorkerRequestHandler.handleRequest(request, requesterIdentity, requestContext);
+ if (request.has("queryOptions")) {
+ Map<String, String> queryOptionMap = BaseBrokerRequestHandler.getOptionsFromJson(request, "queryOptions");
+ if (Boolean.parseBoolean(queryOptionMap.get(
+ CommonConstants.Broker.Request.QueryOptionKey.USE_MULTISTAGE_ENGINE))) {
+ return _multiStageWorkerRequestHandler.handleRequest(request, requesterIdentity, requestContext);
+ }
}
}
return _singleStageBrokerRequestHandler.handleRequest(request, requesterIdentity, requestContext);
diff --git a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerRequestOptionsTest.java b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerRequestOptionsTest.java
index dc83218d08..527e4585da 100644
--- a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerRequestOptionsTest.java
+++ b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerRequestOptionsTest.java
@@ -33,6 +33,8 @@ import org.testng.annotations.Test;
* Tests the various options set in the broker request
*/
public class BrokerRequestOptionsTest {
+ // TODO: remove this legacy option size checker after 0.11 release cut.
+ private static final int LEGACY_PQL_QUERY_OPTION_SIZE = 2;
@Test
public void testSetOptions() {
@@ -44,7 +46,7 @@ public class BrokerRequestOptionsTest {
PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query, jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 0);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 0 + LEGACY_PQL_QUERY_OPTION_SIZE);
// TRACE
// Has trace false
@@ -52,14 +54,14 @@ public class BrokerRequestOptionsTest {
pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query, jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 0);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 0 + LEGACY_PQL_QUERY_OPTION_SIZE);
// Has trace true
jsonRequest.put(Request.TRACE, true);
pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query, jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1 + LEGACY_PQL_QUERY_OPTION_SIZE);
Assert.assertEquals(pinotQuery.getQueryOptions().get(Request.TRACE), "true");
// DEBUG_OPTIONS (debug options will also be included as query options)
@@ -70,7 +72,6 @@ public class BrokerRequestOptionsTest {
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query, jsonRequest);
Assert.assertEquals(pinotQuery.getDebugOptionsSize(), 1);
Assert.assertEquals(pinotQuery.getDebugOptions().get("debugOption1"), "foo");
- Assert.assertEquals(pinotQuery.getQueryOptions(), pinotQuery.getDebugOptions());
// Has multiple debugOptions
jsonRequest.put(Request.DEBUG_OPTIONS, "debugOption1=foo;debugOption2=bar");
@@ -79,7 +80,6 @@ public class BrokerRequestOptionsTest {
Assert.assertEquals(pinotQuery.getDebugOptionsSize(), 2);
Assert.assertEquals(pinotQuery.getDebugOptions().get("debugOption1"), "foo");
Assert.assertEquals(pinotQuery.getDebugOptions().get("debugOption2"), "bar");
- Assert.assertEquals(pinotQuery.getQueryOptions(), pinotQuery.getDebugOptions());
// Invalid debug options
jsonRequest.put(Request.DEBUG_OPTIONS, "debugOption1");
@@ -100,14 +100,14 @@ public class BrokerRequestOptionsTest {
pinotQuery.setQueryOptions(queryOptions);
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query, jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1 + LEGACY_PQL_QUERY_OPTION_SIZE);
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption1"), "foo");
// Has queryOptions in query
pinotQuery = CalciteSqlParser.compileToPinotQuery("SET queryOption1='foo'; select * from testTable");
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query, jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1 + LEGACY_PQL_QUERY_OPTION_SIZE);
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption1"), "foo");
// Has query options in json payload
@@ -115,7 +115,7 @@ public class BrokerRequestOptionsTest {
pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query, jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1 + LEGACY_PQL_QUERY_OPTION_SIZE);
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption1"), "foo");
// Has query options in both json payload and pinotQuery, pinotQuery takes priority
@@ -123,7 +123,7 @@ public class BrokerRequestOptionsTest {
pinotQuery = CalciteSqlParser.compileToPinotQuery("SET queryOption1='foo'; select * from testTable;");
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query, jsonRequest);
Assert.assertNull(pinotQuery.getDebugOptions());
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 2);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 2 + LEGACY_PQL_QUERY_OPTION_SIZE);
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption1"), "foo");
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption2"), "moo");
@@ -136,7 +136,7 @@ public class BrokerRequestOptionsTest {
BaseBrokerRequestHandler.setOptions(pinotQuery, requestId, query, jsonRequest);
Assert.assertEquals(pinotQuery.getDebugOptionsSize(), 1);
Assert.assertEquals(pinotQuery.getDebugOptions().get("debugOption1"), "foo");
- Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 4);
+ Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 4 + LEGACY_PQL_QUERY_OPTION_SIZE);
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption1"), "bar");
Assert.assertEquals(pinotQuery.getQueryOptions().get("queryOption2"), "moo");
Assert.assertEquals(pinotQuery.getQueryOptions().get(Request.TRACE), "true");
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
index a21279c549..a382122c33 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
@@ -29,6 +29,7 @@ import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -48,6 +49,7 @@ import org.apache.commons.lang3.tuple.Pair;
import org.apache.helix.model.InstanceConfig;
import org.apache.pinot.common.Utils;
import org.apache.pinot.common.exception.QueryException;
+import org.apache.pinot.common.utils.config.TagNameUtils;
import org.apache.pinot.common.utils.request.RequestUtils;
import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.controller.api.access.AccessControl;
@@ -55,6 +57,7 @@ import org.apache.pinot.controller.api.access.AccessControlFactory;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.core.query.executor.sql.SqlQueryExecutor;
import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.apache.pinot.sql.parsers.CalciteSqlCompiler;
@@ -104,48 +107,76 @@ public class PinotQueryResource {
}
}
- private String executeSqlQuery(@Context HttpHeaders httpHeaders, String sqlQuery, String traceEnabled,
- String queryOptions)
- throws Exception {
- SqlNodeAndOptions sqlNodeAndOptions = CalciteSqlParser.compileToSqlNodeAndOptions(sqlQuery);
- PinotSqlType sqlType = sqlNodeAndOptions.getSqlType();
- switch (sqlType) {
- case DQL:
- return getQueryResponse(sqlQuery, sqlNodeAndOptions.getSqlNode(), traceEnabled, queryOptions, httpHeaders);
- case DML:
- Map<String, String> headers =
- httpHeaders.getRequestHeaders().entrySet().stream().filter(entry -> !entry.getValue().isEmpty())
- .map(entry -> Pair.of(entry.getKey(), entry.getValue().get(0)))
- .collect(Collectors.toMap(Pair::getKey, Pair::getValue));
- return _sqlQueryExecutor.executeDMLStatement(sqlNodeAndOptions, headers).toJsonString();
- default:
- throw new UnsupportedOperationException("Unsupported SQL type - " + sqlType);
- }
- }
-
@GET
@Path("sql")
public String handleGetSql(@QueryParam("sql") String sqlQuery, @QueryParam("trace") String traceEnabled,
@QueryParam("queryOptions") String queryOptions, @Context HttpHeaders httpHeaders) {
try {
LOGGER.debug("Trace: {}, Running query: {}", traceEnabled, sqlQuery);
- SqlNodeAndOptions sqlNodeAndOptions = CalciteSqlParser.compileToSqlNodeAndOptions(sqlQuery);
- PinotSqlType sqlType = sqlNodeAndOptions.getSqlType();
- if (sqlType == PinotSqlType.DQL) {
- return getQueryResponse(sqlQuery, sqlNodeAndOptions.getSqlNode(), traceEnabled, queryOptions, httpHeaders);
- }
- throw new UnsupportedOperationException("Unsupported SQL type - " + sqlType);
+ return executeSqlQuery(httpHeaders, sqlQuery, traceEnabled, queryOptions);
} catch (Exception e) {
LOGGER.error("Caught exception while processing get request", e);
return QueryException.getException(QueryException.INTERNAL_ERROR, e).toString();
}
}
- public String getQueryResponse(String query, String traceEnabled, String queryOptions, HttpHeaders httpHeaders) {
- return getQueryResponse(query, null, traceEnabled, queryOptions, httpHeaders);
+ private String executeSqlQuery(@Context HttpHeaders httpHeaders, String sqlQuery, String traceEnabled,
+ String queryOptions)
+ throws Exception {
+ if (queryOptions != null && queryOptions.contains(QueryOptionKey.USE_MULTISTAGE_ENGINE)) {
+ if (_controllerConf.getProperty(CommonConstants.Helix.CONFIG_OF_MULTI_STAGE_ENGINE_ENABLED,
+ CommonConstants.Helix.DEFAULT_MULTI_STAGE_ENGINE_ENABLED)) {
+ return getMultiStageQueryResponse(sqlQuery, queryOptions, httpHeaders);
+ } else {
+ throw new UnsupportedOperationException("V2 Multi-Stage query engine not enabled. "
+ + "Please see https://docs.pinot.apache.org/ for instruction to enable V2 engine.");
+ }
+ } else {
+ SqlNodeAndOptions sqlNodeAndOptions = CalciteSqlParser.compileToSqlNodeAndOptions(sqlQuery);
+ PinotSqlType sqlType = sqlNodeAndOptions.getSqlType();
+ switch (sqlType) {
+ case DQL:
+ return getQueryResponse(sqlQuery, sqlNodeAndOptions.getSqlNode(), traceEnabled, queryOptions, httpHeaders);
+ case DML:
+ Map<String, String> headers =
+ httpHeaders.getRequestHeaders().entrySet().stream().filter(entry -> !entry.getValue().isEmpty())
+ .map(entry -> Pair.of(entry.getKey(), entry.getValue().get(0)))
+ .collect(Collectors.toMap(Pair::getKey, Pair::getValue));
+ return _sqlQueryExecutor.executeDMLStatement(sqlNodeAndOptions, headers).toJsonString();
+ default:
+ throw new UnsupportedOperationException("Unsupported SQL type - " + sqlType);
+ }
+ }
+ }
+
+ private String getMultiStageQueryResponse(String query, String queryOptions, HttpHeaders httpHeaders) {
+
+ // Validate data access
+ // we don't have a cross table access control rule so only ADMIN can make request to multi-stage engine.
+ AccessControl accessControl = _accessControlFactory.create();
+ if (!accessControl.hasAccess(httpHeaders)) {
+ return QueryException.ACCESS_DENIED_ERROR.toString();
+ }
+
+ // Get brokers, only DEFAULT tenant is supported for now.
+ // TODO: implement logic that only allows executing query where all accessed tables are within the same tenant.
+ List<String> instanceIds = new ArrayList<>(_pinotHelixResourceManager.getAllInstancesForBrokerTenant(
+ TagNameUtils.DEFAULT_TENANT_NAME));
+ if (instanceIds.isEmpty()) {
+ return QueryException.BROKER_RESOURCE_MISSING_ERROR.toString();
+ }
+
+ instanceIds.retainAll(_pinotHelixResourceManager.getOnlineInstanceList());
+ if (instanceIds.isEmpty()) {
+ return QueryException.BROKER_INSTANCE_MISSING_ERROR.toString();
+ }
+
+ // Send query to a random broker.
+ String instanceId = instanceIds.get(RANDOM.nextInt(instanceIds.size()));
+ return sendRequestToBroker(query, instanceId, "false", queryOptions, httpHeaders);
}
- public String getQueryResponse(String query, @Nullable SqlNode sqlNode, String traceEnabled, String queryOptions,
+ private String getQueryResponse(String query, @Nullable SqlNode sqlNode, String traceEnabled, String queryOptions,
HttpHeaders httpHeaders) {
// Get resource table name.
String tableName;
@@ -180,6 +211,11 @@ public class PinotQueryResource {
// Send query to a random broker.
String instanceId = instanceIds.get(RANDOM.nextInt(instanceIds.size()));
+ return sendRequestToBroker(query, instanceId, traceEnabled, queryOptions, httpHeaders);
+ }
+
+ private String sendRequestToBroker(String query, String instanceId, String traceEnabled, String queryOptions,
+ HttpHeaders httpHeaders) {
InstanceConfig instanceConfig = _pinotHelixResourceManager.getHelixInstanceConfig(instanceId);
if (instanceConfig == null) {
LOGGER.error("Instance {} not found", instanceId);
diff --git a/pinot-controller/src/main/resources/app/pages/Query.tsx b/pinot-controller/src/main/resources/app/pages/Query.tsx
index 3ced676e92..4a6d3f3993 100644
--- a/pinot-controller/src/main/resources/app/pages/Query.tsx
+++ b/pinot-controller/src/main/resources/app/pages/Query.tsx
@@ -194,6 +194,7 @@ const QueryPage = () => {
const [checked, setChecked] = React.useState({
tracing: queryParam.get('tracing') === 'true',
+ useMSE: queryParam.get('useMSE') === 'true',
showResultJSON: false,
});
@@ -270,19 +271,24 @@ const QueryPage = () => {
setQueryLoader(true);
queryExecuted.current = true;
let params;
- let timeoutStr = '';
+ let queryOptions = '';
if(queryTimeout){
- timeoutStr = ` option(timeoutMs=${queryTimeout})`
+ queryOptions += `timeoutMs=${queryTimeout}`;
+ }
+ if(checked.useMSE){
+ queryOptions += `useMultistageEngine=true`;
}
const finalQuery = `${query || inputQuery.trim()}`;
params = JSON.stringify({
- sql: `${finalQuery}${timeoutStr}`,
+ sql: `${finalQuery}`,
trace: checked.tracing,
+ queryOptions: `${queryOptions}`,
});
if(finalQuery !== ''){
queryParam.set('query', finalQuery);
queryParam.set('tracing', checked.tracing.toString());
+ queryParam.set('useMSE', checked.useMSE.toString());
if(queryTimeout !== undefined && queryTimeout !== ''){
queryParam.set('timeout', queryTimeout.toString());
}
@@ -292,7 +298,7 @@ const QueryPage = () => {
})
}
- const results = await PinotMethodUtils.getQueryResults(params, checked);
+ const results = await PinotMethodUtils.getQueryResults(params);
setResultError(results.error || '');
setResultData(results.result || { columns: [], records: [] });
setQueryStats(results.queryStats || { columns: responseStatCols, records: [] });
@@ -310,6 +316,10 @@ const QueryPage = () => {
+ `created with an older schema. `
+ `Please reload the table in order to refresh these segments to the new schema.`);
}
+ if (checked.useMSE) {
+ warnings.push(`Using V2 Multi-Stage Query Engine. This is an experimental feature. Please report any bugs to `
+ + `Apache Pinot Slack channel.`);
+ }
return warnings;
}
@@ -377,6 +387,7 @@ const QueryPage = () => {
setInputQuery(query);
setChecked({
tracing: queryParam.get('tracing') === 'true',
+ useMSE: queryParam.get('useMse') === 'true',
showResultJSON: checked.showResultJSON,
});
setQueryTimeout(Number(queryParam.get('timeout') || '') || '');
@@ -492,6 +503,16 @@ const QueryPage = () => {
Tracing
</Grid>
+ <Grid item xs={2}>
+ <Checkbox
+ name="useMSE"
+ color="primary"
+ onChange={handleChange}
+ checked={checked.useMSE}
+ />
+ Use V2 Engine
+ </Grid>
+
<Grid item xs={3}>
<FormControl fullWidth={true} className={classes.timeoutControl}>
<InputLabel htmlFor="my-input">Timeout (in Milliseconds)</InputLabel>
@@ -525,19 +546,20 @@ const QueryPage = () => {
</Grid>
) : null}
+ {
+ warnings.map(warn =>
+ <Alert severity="warning" className={classes.sqlError}>
+ {warn}
+ </Alert>
+ )
+ }
+
{resultError ? (
<Alert severity="error" className={classes.sqlError}>
{resultError}
</Alert>
) : (
<>
- {
- warnings.map(warn =>
- <Alert severity="warning" className={classes.sqlError}>
- {warn}
- </Alert>
- )
- }
<Grid item xs style={{ backgroundColor: 'white' }}>
{resultData.columns.length ? (
<>
diff --git a/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts b/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
index 0b6816d10b..1c41399ea0 100644
--- a/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
+++ b/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
@@ -254,7 +254,7 @@ const getAsObject = (str: SQLResult) => {
// This method is used to display query output in tabular format as well as JSON format on query page
// API: /:urlName (Eg: sql or pql)
// Expected Output: {columns: [], records: []}
-const getQueryResults = (params, checkedOptions) => {
+const getQueryResults = (params) => {
return getQueryResult(params).then(({ data }) => {
let queryResponse = getAsObject(data);
@@ -268,46 +268,7 @@ const getQueryResults = (params, checkedOptions) => {
errorStr = JSON.stringify(queryResponse.exceptions, null, 2);
} else
{
- if (checkedOptions.querySyntaxPQL === true)
- {
- if (queryResponse)
- {
- if (queryResponse.selectionResults)
- {
- // Selection query
- columnList = queryResponse.selectionResults.columns;
- dataArray = queryResponse.selectionResults.results;
- }
- else if (!queryResponse.aggregationResults[0]?.groupByResult)
- {
- // Simple aggregation query
- columnList = map(queryResponse.aggregationResults, (aggregationResult) => {
- return {title: aggregationResult.function};
- });
-
- dataArray.push(map(queryResponse.aggregationResults, (aggregationResult) => {
- return aggregationResult.value;
- }));
- }
- else if (queryResponse.aggregationResults[0]?.groupByResult)
- {
- // Aggregation group by query
- // TODO - Revisit
- const columns = queryResponse.aggregationResults[0].groupByColumns;
- columns.push(queryResponse.aggregationResults[0].function);
- columnList = map(columns, (columnName) => {
- return columnName;
- });
-
- dataArray = map(queryResponse.aggregationResults[0].groupByResult, (aggregationGroup) => {
- const row = aggregationGroup.group;
- row.push(aggregationGroup.value);
- return row;
- });
- }
- }
- }
- else if (queryResponse.resultTable?.dataSchema?.columnNames?.length)
+ if (queryResponse.resultTable?.dataSchema?.columnNames?.length)
{
columnList = queryResponse.resultTable.dataSchema.columnNames;
dataArray = queryResponse.resultTable.rows;
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
index e5c92c1799..60b2c4211f 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
@@ -102,7 +102,8 @@ public class MultiStageEngineIntegrationTest extends BaseClusterIntegrationTest
public void testMultiStageQuery(String sql, int expectedNumOfRows, int expectedNumOfColumns)
throws IOException {
JsonNode multiStageResponse = JsonUtils.stringToJsonNode(
- sendPostRequest(_brokerBaseApiUrl + "/query/sql", "{\"useMultistageEngine\": true, \"sql\":\"" + sql + "\"}"));
+ sendPostRequest(_brokerBaseApiUrl + "/query/sql",
+ "{\"queryOptions\":\"useMultistageEngine=true\", \"sql\":\"" + sql + "\"}"));
Assert.assertTrue(multiStageResponse.has("resultTable"));
ArrayNode jsonNode = (ArrayNode) multiStageResponse.get("resultTable").get("rows");
// TODO: assert actual result data payload.
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
index c7179c6856..1bbb87bca5 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
@@ -93,11 +93,11 @@ public class MultistageEngineQuickStart extends QuickStartBase {
waitForBootstrapToComplete(null);
- Map<String, String> queryOptions = Collections.singletonMap(
- CommonConstants.Broker.Request.QueryOptionKey.USE_MULTISTAGE_ENGINE, "true");
+ Map<String, String> queryOptions = Collections.singletonMap("queryOptions",
+ CommonConstants.Broker.Request.QueryOptionKey.USE_MULTISTAGE_ENGINE + "=true");
printStatus(Quickstart.Color.YELLOW, "***** Multi-stage engine quickstart setup complete *****");
- String q1 = "SELECT count(*) FROM baseballStats_OFFLINE limit 1";
+ String q1 = "SELECT count(*) FROM baseballStats_OFFLINE";
printStatus(Quickstart.Color.YELLOW, "Total number of documents in the table");
printStatus(Quickstart.Color.CYAN, "Query : " + q1);
printStatus(Quickstart.Color.YELLOW, prettyPrintResponse(runner.runQuery(q1, queryOptions)));
@@ -115,8 +115,8 @@ public class MultistageEngineQuickStart extends QuickStartBase {
printStatus(Quickstart.Color.GREEN, "***************************************************");
printStatus(Quickstart.Color.YELLOW, "Example query run completed.");
printStatus(Quickstart.Color.GREEN, "***************************************************");
- printStatus(Quickstart.Color.YELLOW, "Please use broker port for executing multistage queries.");
- printStatus(Quickstart.Color.GREEN, "***************************************************");
+ printStatus(Quickstart.Color.GREEN,
+ "You can always go to http://localhost:9000 to play around in the query console");
}
public static void main(String[] args)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org