You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ab...@apache.org on 2021/11/25 15:39:10 UTC
[druid] branch master updated: Improve the output of SQL explain message (#11908)
This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new c381cae Improve the output of SQL explain message (#11908)
c381cae is described below
commit c381cae51b93251e8ae6e5bd84d15a6bd30e8d28
Author: Laksh Singla <30...@users.noreply.github.com>
AuthorDate: Thu Nov 25 21:08:33 2021 +0530
Improve the output of SQL explain message (#11908)
Currently, when we try to do EXPLAIN PLAN FOR, it returns the structure of the SQL parsed (via Calcite's internal planner util), which is verbose (since it tries to explain about the nodes in the SQL, instead of the Druid Query), and not representative of the native Druid query which will get executed on the broker side.
This PR aims to change the format when user tries to EXPLAIN PLAN FOR for queries which are executed by converting them into Druid's native queries (i.e. not sys schemas).
---
docs/configuration/index.md | 1 +
.../druid/sql/calcite/planner/DruidPlanner.java | 72 ++++++++-
.../druid/sql/calcite/planner/PlannerConfig.java | 21 ++-
.../druid/sql/calcite/BaseCalciteQueryTest.java | 9 ++
.../apache/druid/sql/calcite/CalciteQueryTest.java | 167 +++++++++++++++++----
.../druid/sql/calcite/CalciteSelectQueryTest.java | 73 ++++++++-
6 files changed, 304 insertions(+), 39 deletions(-)
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 6ffd149..0708c07 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -1834,6 +1834,7 @@ The Druid SQL server is configured through the following properties on the Broke
|`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|false|
|`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator for published segments list if `druid.sql.planner.metadataSegmentCacheEnable` is set to true. Poll period is in milliseconds. |60000|
|`druid.sql.planner.authorizeSystemTablesDirectly`|If true, Druid authorizes queries against any of the system schema tables (`sys` in SQL) as `SYSTEM_TABLE` resources which require `READ` access, in addition to permissions based content filtering.|false|
+|`druid.sql.planner.useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with `useNativeQueryExplain` context key.|false|
|`druid.sql.approxCountDistinct.function`|Implementation to use for the [`APPROX_COUNT_DISTINCT` function](../querying/sql.md#aggregation-functions). Without extensions loaded, the only valid value is `APPROX_COUNT_DISTINCT_BUILTIN` (a HyperLogLog, or HLL, based implementation). If the [DataSketches extension](../development/extensions-core/datasketches-extension.md) is loaded, this can also be `APPROX_COUNT_DISTINCT_DS_HLL` (alternative HLL implementation) or `APPROX_COUNT_DISTINCT_DS_T [...]
> Previous versions of Druid had properties named `druid.sql.planner.maxQueryCount` and `druid.sql.planner.maxSemiJoinRowsInMemory`.
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java
index 2d07239..3a5dbeb 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java
@@ -20,6 +20,9 @@
package org.apache.druid.sql.calcite.planner;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
@@ -63,22 +66,27 @@ import org.apache.calcite.tools.Planner;
import org.apache.calcite.tools.RelConversionException;
import org.apache.calcite.tools.ValidationException;
import org.apache.calcite.util.Pair;
+import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.guava.BaseSequence;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.java.util.common.guava.Sequences;
import org.apache.druid.java.util.emitter.EmittingLogger;
+import org.apache.druid.query.Query;
import org.apache.druid.segment.DimensionHandlerUtils;
import org.apache.druid.server.security.Action;
import org.apache.druid.server.security.Resource;
import org.apache.druid.server.security.ResourceAction;
import org.apache.druid.server.security.ResourceType;
import org.apache.druid.sql.calcite.rel.DruidConvention;
+import org.apache.druid.sql.calcite.rel.DruidQuery;
import org.apache.druid.sql.calcite.rel.DruidRel;
+import org.apache.druid.sql.calcite.rel.DruidUnionRel;
import org.apache.druid.sql.calcite.run.QueryMaker;
import org.apache.druid.sql.calcite.run.QueryMakerFactory;
import javax.annotation.Nullable;
+
import java.io.Closeable;
import java.util.ArrayList;
import java.util.HashSet;
@@ -270,7 +278,7 @@ public class DruidPlanner implements Closeable
);
if (explain != null) {
- return planExplanation(druidRel, explain);
+ return planExplanation(druidRel, explain, true);
} else {
final Supplier<Sequence<Object[]>> resultsSupplier = () -> {
// sanity check
@@ -331,7 +339,7 @@ public class DruidPlanner implements Closeable
}
if (explain != null) {
- return planExplanation(bindableRel, explain);
+ return planExplanation(bindableRel, explain, false);
} else {
final BindableRel theRel = bindableRel;
final DataContext dataContext = plannerContext.createDataContext(
@@ -380,12 +388,20 @@ public class DruidPlanner implements Closeable
*/
private PlannerResult planExplanation(
final RelNode rel,
- final SqlExplain explain
+ final SqlExplain explain,
+ final boolean isDruidConventionExplanation
)
{
- final String explanation = RelOptUtil.dumpPlan("", rel, explain.getFormat(), explain.getDetailLevel());
+ String explanation = RelOptUtil.dumpPlan("", rel, explain.getFormat(), explain.getDetailLevel());
String resourcesString;
try {
+ if (isDruidConventionExplanation && rel instanceof DruidRel) {
+ // Show the native queries instead of Calcite's explain if the legacy flag is turned off
+ if (plannerContext.getPlannerConfig().isUseNativeQueryExplain()) {
+ DruidRel<?> druidRel = (DruidRel<?>) rel;
+ explanation = explainSqlPlanAsNativeQueries(druidRel);
+ }
+ }
final Set<Resource> resources =
plannerContext.getResourceActions().stream().map(ResourceAction::getResource).collect(Collectors.toSet());
resourcesString = plannerContext.getJsonMapper().writeValueAsString(resources);
@@ -395,12 +411,59 @@ public class DruidPlanner implements Closeable
log.error(jpe, "Encountered exception while serializing Resources for explain output");
resourcesString = null;
}
+ catch (ISE ise) {
+ log.error(ise, "Unable to translate to a native Druid query. Resorting to legacy Druid explain plan");
+ resourcesString = null;
+ }
final Supplier<Sequence<Object[]>> resultsSupplier = Suppliers.ofInstance(
Sequences.simple(ImmutableList.of(new Object[]{explanation, resourcesString})));
return new PlannerResult(resultsSupplier, getExplainStructType(rel.getCluster().getTypeFactory()));
}
/**
+ * This method doesn't utilize the Calcite's internal {@link RelOptUtil#dumpPlan} since that tends to be verbose
+ * and not indicative of the native Druid Queries which will get executed
+ * This method assumes that the Planner has converted the RelNodes to DruidRels, and thereby we can implictly cast it
+ *
+ * @param rel Instance of the root {@link DruidRel} which is formed by running the planner transformations on it
+ * @return A string representing an array of native queries that correspond to the given SQL query, in JSON format
+ * @throws JsonProcessingException
+ */
+ private String explainSqlPlanAsNativeQueries(DruidRel<?> rel) throws JsonProcessingException
+ {
+ // Only if rel is an instance of DruidUnionRel, do we run multiple native queries corresponding to single SQL query
+ // Also, DruidUnionRel can only be a top level node, so we don't need to check for this condition in the subsequent
+ // child nodes
+ ObjectMapper jsonMapper = plannerContext.getJsonMapper();
+ List<DruidQuery> druidQueryList;
+ if (rel instanceof DruidUnionRel) {
+ druidQueryList = rel.getInputs().stream().map(childRel -> (DruidRel<?>) childRel).map(childRel -> {
+ if (childRel instanceof DruidUnionRel) {
+ log.error("DruidUnionRel can only be the outermost RelNode. This error shouldn't be encountered");
+ throw new ISE("DruidUnionRel is only supported at the outermost RelNode.");
+ }
+ return childRel.toDruidQuery(false);
+ }).collect(Collectors.toList());
+ } else {
+ druidQueryList = ImmutableList.of(rel.toDruidQuery(false));
+ }
+
+ // Putting the queries as object node in an ArrayNode, since directly returning a list causes issues when
+ // serializing the "queryType"
+ ArrayNode nativeQueriesArrayNode = jsonMapper.createArrayNode();
+
+ for (DruidQuery druidQuery : druidQueryList) {
+ Query<?> nativeQuery = druidQuery.getQuery();
+ ObjectNode objectNode = jsonMapper.createObjectNode();
+ objectNode.put("query", jsonMapper.convertValue(nativeQuery, ObjectNode.class));
+ objectNode.put("signature", jsonMapper.convertValue(druidQuery.getOutputRowSignature(), ArrayNode.class));
+ nativeQueriesArrayNode.add(objectNode);
+ }
+
+ return jsonMapper.writeValueAsString(nativeQueriesArrayNode);
+ }
+
+ /**
* This method wraps the root with a {@link LogicalSort} that applies a limit (no ordering change). If the outer rel
* is already a {@link Sort}, we can merge our outerLimit into it, similar to what is going on in
* {@link org.apache.druid.sql.calcite.rule.SortCollapseRule}.
@@ -409,7 +472,6 @@ public class DruidPlanner implements Closeable
* the web console, allowing it to apply a limit to queries without rewriting the original SQL.
*
* @param root root node
- *
* @return root node wrapped with a limiting logical sort if a limit is specified in the query context.
*/
@Nullable
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java
index 43dfa5a..6203fdc 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java
@@ -33,6 +33,7 @@ public class PlannerConfig
public static final String CTX_KEY_USE_GROUPING_SET_FOR_EXACT_DISTINCT = "useGroupingSetForExactDistinct";
public static final String CTX_KEY_USE_APPROXIMATE_TOPN = "useApproximateTopN";
public static final String CTX_COMPUTE_INNER_JOIN_COST_AS_FILTER = "computeInnerJoinCostAsFilter";
+ public static final String CTX_KEY_USE_NATIVE_QUERY_EXPLAIN = "useNativeQueryExplain";
@JsonProperty
private Period metadataRefreshPeriod = new Period("PT1M");
@@ -70,6 +71,9 @@ public class PlannerConfig
@JsonProperty
private boolean authorizeSystemTablesDirectly = false;
+ @JsonProperty
+ private boolean useNativeQueryExplain = false;
+
public long getMetadataSegmentPollPeriod()
{
return metadataSegmentPollPeriod;
@@ -137,6 +141,11 @@ public class PlannerConfig
return authorizeSystemTablesDirectly;
}
+ public boolean isUseNativeQueryExplain()
+ {
+ return useNativeQueryExplain;
+ }
+
public PlannerConfig withOverrides(final Map<String, Object> context)
{
if (context == null) {
@@ -166,6 +175,11 @@ public class PlannerConfig
CTX_COMPUTE_INNER_JOIN_COST_AS_FILTER,
computeInnerJoinCostAsFilter
);
+ newConfig.useNativeQueryExplain = getContextBoolean(
+ context,
+ CTX_KEY_USE_NATIVE_QUERY_EXPLAIN,
+ isUseNativeQueryExplain()
+ );
newConfig.requireTimeCondition = isRequireTimeCondition();
newConfig.sqlTimeZone = getSqlTimeZone();
newConfig.awaitInitializationOnStart = isAwaitInitializationOnStart();
@@ -213,7 +227,8 @@ public class PlannerConfig
metadataSegmentPollPeriod == that.metadataSegmentPollPeriod &&
serializeComplexValues == that.serializeComplexValues &&
Objects.equals(metadataRefreshPeriod, that.metadataRefreshPeriod) &&
- Objects.equals(sqlTimeZone, that.sqlTimeZone);
+ Objects.equals(sqlTimeZone, that.sqlTimeZone) &&
+ useNativeQueryExplain == that.useNativeQueryExplain;
}
@Override
@@ -230,7 +245,8 @@ public class PlannerConfig
sqlTimeZone,
metadataSegmentCacheEnable,
metadataSegmentPollPeriod,
- serializeComplexValues
+ serializeComplexValues,
+ useNativeQueryExplain
);
}
@@ -248,6 +264,7 @@ public class PlannerConfig
", metadataSegmentPollPeriod=" + metadataSegmentPollPeriod +
", sqlTimeZone=" + sqlTimeZone +
", serializeComplexValues=" + serializeComplexValues +
+ ", useNativeQueryExplain=" + useNativeQueryExplain +
'}';
}
}
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
index 3e26507..ff85165 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
@@ -191,6 +191,15 @@ public class BaseCalciteQueryTest extends CalciteTestBase
}
};
+ public static final PlannerConfig PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN = new PlannerConfig()
+ {
+ @Override
+ public boolean isUseNativeQueryExplain()
+ {
+ return true;
+ }
+ };
+
public static final String DUMMY_SQL_ID = "dummy";
public static final String LOS_ANGELES = "America/Los_Angeles";
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index 2feda99..e2b0a24 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -4185,25 +4185,35 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
// Skip vectorization since otherwise the "context" will change for each subtest.
skipVectorize();
- final String explanation =
- "DruidQueryRel(query=[{"
- + "\"queryType\":\"timeseries\","
- + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
- + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
- + "\"descending\":false,"
- + "\"virtualColumns\":[],"
- + "\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]},"
- + "\"granularity\":{\"type\":\"all\"},"
- + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],"
- + "\"postAggregations\":[],"
- + "\"limit\":2147483647,"
- + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}]"
- + ", signature=[{a0:LONG}])\n";
-
+ final String query = "EXPLAIN PLAN FOR SELECT COUNT(*) FROM view.aview WHERE dim1_firstchar <> 'z'";
+ final String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimensio [...]
+ final String explanation = "[{"
+ + "\"query\":{\"queryType\":\"timeseries\","
+ + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"descending\":false,"
+ + "\"virtualColumns\":[],"
+ + "\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]},"
+ + "\"granularity\":{\"type\":\"all\"},"
+ + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],"
+ + "\"postAggregations\":[],"
+ + "\"limit\":2147483647,"
+ + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}},"
+ + "\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]"
+ + "}]";
final String resources = "[{\"name\":\"aview\",\"type\":\"VIEW\"}]";
testQuery(
- "EXPLAIN PLAN FOR SELECT COUNT(*) FROM view.aview WHERE dim1_firstchar <> 'z'",
+ query,
+ ImmutableList.of(),
+ ImmutableList.of(
+ new Object[]{legacyExplanation, resources}
+ )
+ );
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ query,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
ImmutableList.of(),
ImmutableList.of(
new Object[]{explanation, resources}
@@ -6739,28 +6749,133 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
// Skip vectorization since otherwise the "context" will change for each subtest.
skipVectorize();
- final String explanation =
+ final String query = "EXPLAIN PLAN FOR SELECT COUNT(*)\n"
+ + "FROM (\n"
+ + " SELECT DISTINCT dim2\n"
+ + " FROM druid.foo\n"
+ + " WHERE SUBSTRING(dim2, 1, 1) IN (\n"
+ + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo WHERE dim1 IS NOT NULL\n"
+ + " )\n"
+ + ")";
+ final String legacyExplanation =
"DruidOuterQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"defaultTimeout\":300000,\"maxScatte [...]
+ " DruidJoinQueryRel(condition=[=(SUBSTRING($3, 1, 1), $8)], joinType=[inner], query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"__join__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggre [...]
+ " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBy [...]
+ " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputNam [...]
-
+ final String explanation = "["
+ + "{\"query\":{\"queryType\":\"groupBy\","
+ + "\"dataSource\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"join\",\"left\":{\"type\":\"table\",\"name\":\"foo\"},\"right\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"t [...]
+ + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":[],"
+ + "\"filter\":null,"
+ + "\"granularity\":{\"type\":\"all\"},"
+ + "\"dimensions\":[],"
+ + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],"
+ + "\"postAggregations\":[],"
+ + "\"having\":null,"
+ + "\"limitSpec\":{\"type\":\"NoopLimitSpec\"},"
+ + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"descending\":false},"
+ + "\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]"
+ + "}]";
final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
testQuery(
- "EXPLAIN PLAN FOR SELECT COUNT(*)\n"
- + "FROM (\n"
- + " SELECT DISTINCT dim2\n"
- + " FROM druid.foo\n"
- + " WHERE SUBSTRING(dim2, 1, 1) IN (\n"
- + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo WHERE dim1 IS NOT NULL\n"
- + " )\n"
- + ")",
+ query,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{legacyExplanation, resources})
+ );
+
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ query,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
ImmutableList.of(),
ImmutableList.of(new Object[]{explanation, resources})
);
}
+ // This testcase has been added here and not in CalciteSelectQueryTests since this checks if the overrides are working
+ // properly when displaying the output of "EXPLAIN PLAN FOR ..." queries
+ @Test
+ public void testExplainSelectStarWithOverrides() throws Exception
+ {
+ Map<String, Object> useRegularExplainContext = new HashMap<>(QUERY_CONTEXT_DEFAULT);
+ useRegularExplainContext.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, false);
+
+ Map<String, Object> useNativeQueryExplain = new HashMap<>(QUERY_CONTEXT_DEFAULT);
+ useNativeQueryExplain.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, true);
+
+
+ // Skip vectorization since otherwise the "context" will change for each subtest.
+ skipVectorize();
+ String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"m [...]
+ String legacyExplanationWithContext = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\" [...]
+ String explanation = "[{"
+ + "\"query\":{\"queryType\":\"scan\","
+ + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":[],"
+ + "\"resultFormat\":\"compactedList\","
+ + "\"batchSize\":20480,"
+ + "\"filter\":null,"
+ + "\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],"
+ + "\"legacy\":false,"
+ + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"descending\":false,"
+ + "\"granularity\":{\"type\":\"all\"}},"
+ + "\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX<hyperUnique>\"}]"
+ + "}]";
+
+ String explanationWithContext = "[{"
+ + "\"query\":{\"queryType\":\"scan\","
+ + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":[],"
+ + "\"resultFormat\":\"compactedList\","
+ + "\"batchSize\":20480,"
+ + "\"filter\":null,"
+ + "\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],"
+ + "\"legacy\":false,"
+ + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useNativeQueryExplain\":true,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"descending\":false,"
+ + "\"granularity\":{\"type\":\"all\"}},"
+ + "\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX<hyperUnique>\"}]"
+ + "}]";
+ String sql = "EXPLAIN PLAN FOR SELECT * FROM druid.foo";
+ String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
+
+ // Test when default config and no overrides
+ testQuery(sql, ImmutableList.of(), ImmutableList.of(new Object[]{legacyExplanation, resources}));
+
+ // Test when default config and useNativeQueryExplain is overridden in the context
+ testQuery(
+ sql,
+ useNativeQueryExplain,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{explanationWithContext, resources})
+ );
+
+ // Test when useNativeQueryExplain enabled by default and no overrides
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ sql,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{explanation, resources})
+ );
+
+ // Test when useNativeQueryExplain enabled by default but is overriden in the context
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ useRegularExplainContext,
+ sql,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
+ ImmutableList.of(),
+ ImmutableList.of(new Object[]{legacyExplanationWithContext, resources})
+ );
+ }
+
@Test
public void testExactCountDistinctUsingSubqueryWithWherePushDown() throws Exception
{
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
index c6f9808..a4aaab8 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
@@ -409,14 +409,44 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
{
// Skip vectorization since otherwise the "context" will change for each subtest.
skipVectorize();
+ final String query = "EXPLAIN PLAN FOR SELECT 1 + 1";
+ final String explanation = "[{"
+ + "\"query\":{\"queryType\":\"scan\","
+ + "\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},"
+ + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":[],"
+ + "\"resultFormat\":\"compactedList\","
+ + "\"batchSize\":20480,"
+ + "\"filter\":null,"
+ + "\"columns\":[\"EXPR$0\"],"
+ + "\"legacy\":false,"
+ + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"descending\":false,"
+ + "\"granularity\":{\"type\":\"all\"}},"
+ + "\"signature\":[{\"name\":\"EXPR$0\",\"type\":\"LONG\"}]"
+ + "}]";
+ final String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxSc [...]
+ final String resources = "[]";
testQuery(
- "EXPLAIN PLAN FOR SELECT 1 + 1",
+ query,
ImmutableList.of(),
ImmutableList.of(
new Object[]{
- "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":92 [...]
- "[]"
+ legacyExplanation,
+ resources
+ }
+ )
+ );
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ query,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
+ ImmutableList.of(),
+ ImmutableList.of(
+ new Object[]{
+ explanation,
+ resources
}
)
);
@@ -1078,13 +1108,44 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
// Skip vectorization since otherwise the "context" will change for each subtest.
skipVectorize();
+ final String query = "EXPLAIN PLAN FOR SELECT * FROM druid.foo";
+ final String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":3000 [...]
+ final String explanation = "[{"
+ + "\"query\":{\"queryType\":\"scan\","
+ + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
+ + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},"
+ + "\"virtualColumns\":[],"
+ + "\"resultFormat\":\"compactedList\","
+ + "\"batchSize\":20480,"
+ + "\"filter\":null,"
+ + "\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],"
+ + "\"legacy\":false,"
+ + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},"
+ + "\"descending\":false,"
+ + "\"granularity\":{\"type\":\"all\"}},"
+ + "\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX<hyperUnique>\"}]"
+ + "}]";
+ final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
+
testQuery(
- "EXPLAIN PLAN FOR SELECT * FROM druid.foo",
+ query,
+ ImmutableList.of(),
+ ImmutableList.of(
+ new Object[]{
+ legacyExplanation,
+ resources
+ }
+ )
+ );
+ testQuery(
+ PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN,
+ query,
+ CalciteTests.REGULAR_USER_AUTH_RESULT,
ImmutableList.of(),
ImmutableList.of(
new Object[]{
- "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGather [...]
- "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"
+ explanation,
+ resources
}
)
);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org