You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by vo...@apache.org on 2022/08/15 06:31:28 UTC
[druid] branch master updated: SQL: Morph QueryMakerFactory into SqlEngine. (#12897)
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 6c5a43106a SQL: Morph QueryMakerFactory into SqlEngine. (#12897)
6c5a43106a is described below
commit 6c5a43106a1f7d0b0258b943c51b93c52563e9ed
Author: Gian Merlino <gi...@gmail.com>
AuthorDate: Sun Aug 14 23:31:19 2022 -0700
SQL: Morph QueryMakerFactory into SqlEngine. (#12897)
* SQL: Morph QueryMakerFactory into SqlEngine.
Groundwork for introducing an indexing-service-task-based SQL engine
under the umbrella of #12262. Also includes some other changes related
to improving error behavior.
Main changes:
1) Elevate the QueryMakerFactory interface (an extension point that allows
customization of how queries are made) into SqlEngine. SQL engines
can influence planner behavior through EngineFeatures, and can fully
control the mechanics of query execution using QueryMakers.
2) Remove the server-wide QueryMakerFactory choice, in favor of the choice
being made by the SQL entrypoint. The indexing-service-task-based
SQL engine would be associated with its own entrypoint, like
/druid/v2/sql/task.
Other changes:
1) Adjust DruidPlanner to try either DRUID or BINDABLE convention based
on analysis of the planned rels; never try both. In particular, we
no longer try BINDABLE when DRUID fails. This simplifies the logic
and improves error messages.
2) Adjust error message "Cannot build plan for query" to omit the SQL
query text. Useful because the text can be quite long, which makes it
easy to miss the text about the problem.
3) Add a feature to block context parameters used internally by the SQL
planner from being supplied by end users.
4) Add a feature to enable adding row signature to the context for
Scan queries. This is useful in building the task-based engine.
5) Add saffron.properties file that turns off sets and graphviz dumps
in "cannot plan" errors. Significantly reduces log spam on the Broker.
* Fixes from CI.
* Changes from review.
* Can vectorize, now that join-to-filter is on by default.
* Checkstyle! And variable renames!
* Remove throws from test.
---
.../apache/druid/benchmark/query/SqlBenchmark.java | 9 +-
.../benchmark/query/SqlExpressionBenchmark.java | 7 +-
.../benchmark/query/SqlNestedDataBenchmark.java | 8 +-
.../benchmark/query/SqlVsNativeBenchmark.java | 7 +-
.../java/org/apache/druid/sql/DirectStatement.java | 1 +
.../java/org/apache/druid/sql/HttpStatement.java | 1 -
.../org/apache/druid/sql/PreparedStatement.java | 11 +-
.../org/apache/druid/sql/SqlStatementFactory.java | 54 +------
...actory.java => SqlStatementFactoryFactory.java} | 52 ++++--
.../main/java/org/apache/druid/sql/SqlToolbox.java | 18 +++
.../sql/avatica/AbstractDruidJdbcStatement.java | 2 +-
.../apache/druid/sql/avatica/DruidConnection.java | 8 +-
.../org/apache/druid/sql/avatica/DruidMeta.java | 51 ++++--
.../calcite/external/ExternalTableScanRule.java | 16 +-
.../sql/calcite/planner/CalcitePlannerModule.java | 19 ---
.../apache/druid/sql/calcite/planner/Calcites.java | 1 -
.../druid/sql/calcite/planner/DruidPlanner.java | 176 ++++++++++++++++-----
.../druid/sql/calcite/planner/PlannerContext.java | 17 ++
.../druid/sql/calcite/planner/PlannerFactory.java | 15 +-
.../druid/sql/calcite/planner/PlannerResult.java | 3 +
.../druid/sql/calcite/planner/PrepareResult.java | 36 ++++-
.../apache/druid/sql/calcite/rel/DruidQuery.java | 104 +++++++++---
.../run/{QueryFeature.java => EngineFeature.java} | 44 +++++-
.../druid/sql/calcite/run/NativeQueryMaker.java | 31 +---
.../sql/calcite/run/NativeQueryMakerFactory.java | 69 --------
.../druid/sql/calcite/run/NativeSqlEngine.java | 138 ++++++++++++++++
.../sql/calcite/run/QueryFeatureInspector.java | 32 ----
.../apache/druid/sql/calcite/run/QueryMaker.java | 15 +-
.../druid/sql/calcite/run/QueryMakerFactory.java | 60 -------
.../apache/druid/sql/calcite/run/SqlEngine.java | 98 ++++++++++++
.../apache/druid/sql/calcite/run/SqlEngines.java | 52 ++++++
.../druid/sql/calcite/view/DruidViewMacro.java | 5 +-
.../druid/sql/calcite/view/ViewSqlEngine.java | 113 +++++++++++++
.../org/apache/druid/sql/http/SqlResource.java | 33 +++-
sql/src/main/resources/saffron.properties | 5 +-
.../org/apache/druid/sql/SqlStatementTest.java | 38 +++--
.../druid/sql/avatica/DruidAvaticaHandlerTest.java | 48 +++---
.../druid/sql/avatica/DruidStatementTest.java | 12 +-
.../druid/sql/calcite/BaseCalciteQueryTest.java | 111 +++++++++----
.../druid/sql/calcite/CalciteExplainQueryTest.java | 2 +-
.../druid/sql/calcite/CalciteIngestionDmlTest.java | 9 +-
.../druid/sql/calcite/CalciteInsertDmlTest.java | 119 ++++++++++++--
.../apache/druid/sql/calcite/CalciteQueryTest.java | 50 ++++--
.../druid/sql/calcite/CalciteReplaceDmlTest.java | 29 ++--
.../sql/calcite/CalciteScanSignatureTest.java | 142 +++++++++++++++++
.../druid/sql/calcite/IngestionTestSqlEngine.java | 112 +++++++++++++
.../calcite/SqlVectorizedExpressionSanityTest.java | 27 ++--
.../druid/sql/calcite/TestInsertQueryMaker.java | 43 -----
.../druid/sql/calcite/TestQueryMakerFactory.java | 62 --------
.../calcite/expression/ExpressionTestHelper.java | 2 +-
.../external/ExternalTableScanRuleTest.java | 17 +-
.../calcite/planner/CalcitePlannerModuleTest.java | 5 +-
.../sql/calcite/planner/DruidRexExecutorTest.java | 1 +
.../druid/sql/calcite/util/CalciteTests.java | 24 +--
.../org/apache/druid/sql/guice/SqlModuleTest.java | 1 -
.../apache/druid/sql/http/SqlHttpModuleTest.java | 17 +-
.../org/apache/druid/sql/http/SqlResourceTest.java | 96 ++++++++---
57 files changed, 1582 insertions(+), 696 deletions(-)
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java
index aa679b3555..8b65ba0ebe 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java
@@ -29,6 +29,7 @@ import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.query.QueryContext;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.aggregation.datasketches.hll.sql.HllSketchApproxCountDistinctSqlAggregator;
@@ -53,6 +54,7 @@ import org.apache.druid.sql.calcite.planner.DruidPlanner;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.planner.PlannerFactory;
import org.apache.druid.sql.calcite.planner.PlannerResult;
+import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.util.CalciteTests;
import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker;
@@ -405,6 +407,7 @@ public class SqlBenchmark
@Param({STORAGE_MMAP, STORAGE_FRAME_ROW, STORAGE_FRAME_COLUMNAR})
private String storageType;
+ private SqlEngine engine;
@Nullable
private PlannerFactory plannerFactory;
private final Closer closer = Closer.create();
@@ -436,9 +439,9 @@ public class SqlBenchmark
final DruidSchemaCatalog rootSchema =
CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
+ engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
plannerFactory = new PlannerFactory(
rootSchema,
- CalciteTests.createMockQueryMakerFactory(walker, conglomerate),
createOperatorTable(),
CalciteTests.createExprMacroTable(),
plannerConfig,
@@ -513,7 +516,7 @@ public class SqlBenchmark
QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize
);
final String sql = QUERIES.get(Integer.parseInt(query));
- try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(context, sql)) {
+ try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, new QueryContext(context))) {
final PlannerResult plannerResult = planner.plan();
final Sequence<Object[]> resultSequence = plannerResult.run();
final Object[] lastRow = resultSequence.accumulate(null, (accumulated, in) -> in);
@@ -531,7 +534,7 @@ public class SqlBenchmark
QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize
);
final String sql = QUERIES.get(Integer.parseInt(query));
- try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(context, sql)) {
+ try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, new QueryContext(context))) {
final PlannerResult plannerResult = planner.plan();
blackhole.consume(plannerResult);
}
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
index f22832fd6d..f7e3a3a7a4 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
@@ -29,6 +29,7 @@ import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.DruidProcessingConfig;
+import org.apache.druid.query.QueryContext;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.segment.QueryableIndex;
@@ -43,6 +44,7 @@ import org.apache.druid.sql.calcite.planner.DruidPlanner;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.planner.PlannerFactory;
import org.apache.druid.sql.calcite.planner.PlannerResult;
+import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.util.CalciteTests;
import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker;
@@ -274,6 +276,7 @@ public class SqlExpressionBenchmark
})
private String query;
+ private SqlEngine engine;
@Nullable
private PlannerFactory plannerFactory;
private Closer closer = Closer.create();
@@ -310,9 +313,9 @@ public class SqlExpressionBenchmark
final DruidSchemaCatalog rootSchema =
CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
+ engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
plannerFactory = new PlannerFactory(
rootSchema,
- CalciteTests.createMockQueryMakerFactory(walker, conglomerate),
CalciteTests.createOperatorTable(),
CalciteTests.createExprMacroTable(),
plannerConfig,
@@ -349,7 +352,7 @@ public class SqlExpressionBenchmark
QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize
);
final String sql = QUERIES.get(Integer.parseInt(query));
- try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(context, sql)) {
+ try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, new QueryContext(context))) {
final PlannerResult plannerResult = planner.plan();
final Sequence<Object[]> resultSequence = plannerResult.run();
final Object[] lastRow = resultSequence.accumulate(null, (accumulated, in) -> in);
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
index 9eea985d4a..a4876421c6 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
@@ -31,6 +31,7 @@ import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.DruidProcessingConfig;
+import org.apache.druid.query.QueryContext;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.expression.TestExprMacroTable;
@@ -49,6 +50,7 @@ import org.apache.druid.sql.calcite.planner.DruidPlanner;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.planner.PlannerFactory;
import org.apache.druid.sql.calcite.planner.PlannerResult;
+import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.util.CalciteTests;
import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker;
@@ -216,6 +218,7 @@ public class SqlNestedDataBenchmark
})
private String query;
+ private SqlEngine engine;
@Nullable
private PlannerFactory plannerFactory;
private Closer closer = Closer.create();
@@ -234,7 +237,6 @@ public class SqlNestedDataBenchmark
.build();
-
final PlannerConfig plannerConfig = new PlannerConfig();
final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator());
@@ -277,9 +279,9 @@ public class SqlNestedDataBenchmark
final DruidSchemaCatalog rootSchema =
CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
+ engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
plannerFactory = new PlannerFactory(
rootSchema,
- CalciteTests.createMockQueryMakerFactory(walker, conglomerate),
CalciteTests.createOperatorTable(),
CalciteTests.createExprMacroTable(),
plannerConfig,
@@ -316,7 +318,7 @@ public class SqlNestedDataBenchmark
QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize
);
final String sql = QUERIES.get(Integer.parseInt(query));
- try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(context, sql)) {
+ try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, new QueryContext(context))) {
final PlannerResult plannerResult = planner.plan();
final Sequence<Object[]> resultSequence = plannerResult.run();
final Object[] lastRow = resultSequence.accumulate(null, (accumulated, in) -> in);
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java
index 20db919354..d7d2d7ad2c 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java
@@ -26,6 +26,7 @@ import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.query.QueryContext;
import org.apache.druid.query.QueryPlus;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
@@ -44,6 +45,7 @@ import org.apache.druid.sql.calcite.planner.DruidPlanner;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.planner.PlannerFactory;
import org.apache.druid.sql.calcite.planner.PlannerResult;
+import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.util.CalciteTests;
import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker;
@@ -85,6 +87,7 @@ public class SqlVsNativeBenchmark
}
private SpecificSegmentsQuerySegmentWalker walker;
+ private SqlEngine engine;
private PlannerFactory plannerFactory;
private GroupByQuery groupByQuery;
private String sqlQuery;
@@ -115,9 +118,9 @@ public class SqlVsNativeBenchmark
this.walker = closer.register(new SpecificSegmentsQuerySegmentWalker(conglomerate).add(dataSegment, index));
final DruidSchemaCatalog rootSchema =
CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
+ engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
plannerFactory = new PlannerFactory(
rootSchema,
- CalciteTests.createMockQueryMakerFactory(walker, conglomerate),
CalciteTests.createOperatorTable(),
CalciteTests.createExprMacroTable(),
plannerConfig,
@@ -164,7 +167,7 @@ public class SqlVsNativeBenchmark
@OutputTimeUnit(TimeUnit.MILLISECONDS)
public void queryPlanner(Blackhole blackhole) throws Exception
{
- try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(null, sqlQuery)) {
+ try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sqlQuery, new QueryContext())) {
final PlannerResult plannerResult = planner.plan();
final Sequence<Object[]> resultSequence = plannerResult.run();
final Object[] lastRow = resultSequence.accumulate(null, (accumulated, in) -> in);
diff --git a/sql/src/main/java/org/apache/druid/sql/DirectStatement.java b/sql/src/main/java/org/apache/druid/sql/DirectStatement.java
index f6aade646a..a8931bf73d 100644
--- a/sql/src/main/java/org/apache/druid/sql/DirectStatement.java
+++ b/sql/src/main/java/org/apache/druid/sql/DirectStatement.java
@@ -104,6 +104,7 @@ public class DirectStatement extends AbstractStatement implements Cancelable
public Sequence<Object[]> execute()
{
try (DruidPlanner planner = sqlToolbox.plannerFactory.createPlanner(
+ sqlToolbox.engine,
queryPlus.sql(),
queryPlus.context())) {
validate(planner);
diff --git a/sql/src/main/java/org/apache/druid/sql/HttpStatement.java b/sql/src/main/java/org/apache/druid/sql/HttpStatement.java
index 57bb81d8cd..31abd15a42 100644
--- a/sql/src/main/java/org/apache/druid/sql/HttpStatement.java
+++ b/sql/src/main/java/org/apache/druid/sql/HttpStatement.java
@@ -25,7 +25,6 @@ import org.apache.druid.server.security.ResourceAction;
import org.apache.druid.sql.http.SqlQuery;
import javax.servlet.http.HttpServletRequest;
-
import java.util.Set;
import java.util.function.Function;
diff --git a/sql/src/main/java/org/apache/druid/sql/PreparedStatement.java b/sql/src/main/java/org/apache/druid/sql/PreparedStatement.java
index 145f697bf4..b68d6160d3 100644
--- a/sql/src/main/java/org/apache/druid/sql/PreparedStatement.java
+++ b/sql/src/main/java/org/apache/druid/sql/PreparedStatement.java
@@ -20,7 +20,6 @@
package org.apache.druid.sql;
import org.apache.calcite.avatica.remote.TypedValue;
-import org.apache.calcite.tools.ValidationException;
import org.apache.druid.sql.calcite.planner.DruidPlanner;
import org.apache.druid.sql.calcite.planner.PrepareResult;
@@ -67,19 +66,15 @@ public class PreparedStatement extends AbstractStatement
public PrepareResult prepare()
{
try (DruidPlanner planner = sqlToolbox.plannerFactory.createPlanner(
+ sqlToolbox.engine,
queryPlus.sql(),
queryPlus.context())) {
validate(planner);
authorize(planner, authorizer());
// Do the prepare step.
- try {
- this.prepareResult = planner.prepare();
- return prepareResult;
- }
- catch (ValidationException e) {
- throw new SqlPlanningException(e);
- }
+ this.prepareResult = planner.prepare();
+ return prepareResult;
}
catch (RuntimeException e) {
reporter.failed(e);
diff --git a/sql/src/main/java/org/apache/druid/sql/SqlStatementFactory.java b/sql/src/main/java/org/apache/druid/sql/SqlStatementFactory.java
index f1481b8d1f..b1ecde6f4c 100644
--- a/sql/src/main/java/org/apache/druid/sql/SqlStatementFactory.java
+++ b/sql/src/main/java/org/apache/druid/sql/SqlStatementFactory.java
@@ -19,61 +19,15 @@
package org.apache.druid.sql;
-import com.google.common.base.Supplier;
-import com.google.inject.Inject;
-import org.apache.druid.guice.LazySingleton;
-import org.apache.druid.java.util.emitter.service.ServiceEmitter;
-import org.apache.druid.query.DefaultQueryConfig;
-import org.apache.druid.server.QueryScheduler;
-import org.apache.druid.server.log.RequestLogger;
-import org.apache.druid.server.security.AuthConfig;
-import org.apache.druid.sql.calcite.planner.PlannerFactory;
import org.apache.druid.sql.http.SqlQuery;
import javax.servlet.http.HttpServletRequest;
-@LazySingleton
-public class SqlStatementFactory
+public interface SqlStatementFactory
{
- protected final SqlToolbox lifecycleToolbox;
+ HttpStatement httpStatement(SqlQuery sqlQuery, HttpServletRequest req);
- @Inject
- public SqlStatementFactory(
- final PlannerFactory plannerFactory,
- final ServiceEmitter emitter,
- final RequestLogger requestLogger,
- final QueryScheduler queryScheduler,
- final AuthConfig authConfig,
- final Supplier<DefaultQueryConfig> defaultQueryConfig,
- final SqlLifecycleManager sqlLifecycleManager
- )
- {
- this.lifecycleToolbox = new SqlToolbox(
- plannerFactory,
- emitter,
- requestLogger,
- queryScheduler,
- authConfig,
- defaultQueryConfig.get(),
- sqlLifecycleManager
- );
- }
+ DirectStatement directStatement(SqlQueryPlus sqlRequest);
- public HttpStatement httpStatement(
- final SqlQuery sqlQuery,
- final HttpServletRequest req
- )
- {
- return new HttpStatement(lifecycleToolbox, sqlQuery, req);
- }
-
- public DirectStatement directStatement(final SqlQueryPlus sqlRequest)
- {
- return new DirectStatement(lifecycleToolbox, sqlRequest);
- }
-
- public PreparedStatement preparedStatement(final SqlQueryPlus sqlRequest)
- {
- return new PreparedStatement(lifecycleToolbox, sqlRequest);
- }
+ PreparedStatement preparedStatement(SqlQueryPlus sqlRequest);
}
diff --git a/sql/src/main/java/org/apache/druid/sql/SqlStatementFactory.java b/sql/src/main/java/org/apache/druid/sql/SqlStatementFactoryFactory.java
similarity index 60%
copy from sql/src/main/java/org/apache/druid/sql/SqlStatementFactory.java
copy to sql/src/main/java/org/apache/druid/sql/SqlStatementFactoryFactory.java
index f1481b8d1f..5d32735cfe 100644
--- a/sql/src/main/java/org/apache/druid/sql/SqlStatementFactory.java
+++ b/sql/src/main/java/org/apache/druid/sql/SqlStatementFactoryFactory.java
@@ -28,17 +28,24 @@ import org.apache.druid.server.QueryScheduler;
import org.apache.druid.server.log.RequestLogger;
import org.apache.druid.server.security.AuthConfig;
import org.apache.druid.sql.calcite.planner.PlannerFactory;
+import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.http.SqlQuery;
import javax.servlet.http.HttpServletRequest;
+/**
+ * Factory factories: when design patterns go too far.
+ *
+ * Almost everything we need to create a {@link SqlStatementFactory} is injectable, except for the {@link SqlEngine}.
+ * So this class exists to produce {@link SqlStatementFactory} once the engine for a query is known.
+ */
@LazySingleton
-public class SqlStatementFactory
+public class SqlStatementFactoryFactory
{
protected final SqlToolbox lifecycleToolbox;
@Inject
- public SqlStatementFactory(
+ public SqlStatementFactoryFactory(
final PlannerFactory plannerFactory,
final ServiceEmitter emitter,
final RequestLogger requestLogger,
@@ -49,6 +56,7 @@ public class SqlStatementFactory
)
{
this.lifecycleToolbox = new SqlToolbox(
+ null,
plannerFactory,
emitter,
requestLogger,
@@ -59,21 +67,39 @@ public class SqlStatementFactory
);
}
- public HttpStatement httpStatement(
- final SqlQuery sqlQuery,
- final HttpServletRequest req
- )
+ public SqlStatementFactory factorize(final SqlEngine engine)
{
- return new HttpStatement(lifecycleToolbox, sqlQuery, req);
+ return new FactoryImpl(lifecycleToolbox.withEngine(engine));
}
- public DirectStatement directStatement(final SqlQueryPlus sqlRequest)
+ private static class FactoryImpl implements SqlStatementFactory
{
- return new DirectStatement(lifecycleToolbox, sqlRequest);
- }
+ private final SqlToolbox lifecycleToolbox;
- public PreparedStatement preparedStatement(final SqlQueryPlus sqlRequest)
- {
- return new PreparedStatement(lifecycleToolbox, sqlRequest);
+ public FactoryImpl(SqlToolbox lifecycleToolbox)
+ {
+ this.lifecycleToolbox = lifecycleToolbox;
+ }
+
+ @Override
+ public HttpStatement httpStatement(
+ final SqlQuery sqlQuery,
+ final HttpServletRequest req
+ )
+ {
+ return new HttpStatement(lifecycleToolbox, sqlQuery, req);
+ }
+
+ @Override
+ public DirectStatement directStatement(final SqlQueryPlus sqlRequest)
+ {
+ return new DirectStatement(lifecycleToolbox, sqlRequest);
+ }
+
+ @Override
+ public PreparedStatement preparedStatement(final SqlQueryPlus sqlRequest)
+ {
+ return new PreparedStatement(lifecycleToolbox, sqlRequest);
+ }
}
}
diff --git a/sql/src/main/java/org/apache/druid/sql/SqlToolbox.java b/sql/src/main/java/org/apache/druid/sql/SqlToolbox.java
index 8ce9c7a3cd..abbac66829 100644
--- a/sql/src/main/java/org/apache/druid/sql/SqlToolbox.java
+++ b/sql/src/main/java/org/apache/druid/sql/SqlToolbox.java
@@ -26,12 +26,14 @@ import org.apache.druid.server.QueryScheduler;
import org.apache.druid.server.log.RequestLogger;
import org.apache.druid.server.security.AuthConfig;
import org.apache.druid.sql.calcite.planner.PlannerFactory;
+import org.apache.druid.sql.calcite.run.SqlEngine;
/**
* Provides the plan and execution resources to process SQL queries.
*/
public class SqlToolbox
{
+ final SqlEngine engine;
final PlannerFactory plannerFactory;
final ServiceEmitter emitter;
final RequestLogger requestLogger;
@@ -41,6 +43,7 @@ public class SqlToolbox
final SqlLifecycleManager sqlLifecycleManager;
public SqlToolbox(
+ final SqlEngine engine,
final PlannerFactory plannerFactory,
final ServiceEmitter emitter,
final RequestLogger requestLogger,
@@ -50,6 +53,7 @@ public class SqlToolbox
final SqlLifecycleManager sqlLifecycleManager
)
{
+ this.engine = engine;
this.plannerFactory = plannerFactory;
this.emitter = emitter;
this.requestLogger = requestLogger;
@@ -58,4 +62,18 @@ public class SqlToolbox
this.defaultQueryConfig = defaultQueryConfig;
this.sqlLifecycleManager = Preconditions.checkNotNull(sqlLifecycleManager, "sqlLifecycleManager");
}
+
+ public SqlToolbox withEngine(final SqlEngine engine)
+ {
+ return new SqlToolbox(
+ engine,
+ plannerFactory,
+ emitter,
+ requestLogger,
+ queryScheduler,
+ authConfig,
+ defaultQueryConfig,
+ sqlLifecycleManager
+ );
+ }
}
diff --git a/sql/src/main/java/org/apache/druid/sql/avatica/AbstractDruidJdbcStatement.java b/sql/src/main/java/org/apache/druid/sql/avatica/AbstractDruidJdbcStatement.java
index ff7bd91660..c23298463b 100644
--- a/sql/src/main/java/org/apache/druid/sql/avatica/AbstractDruidJdbcStatement.java
+++ b/sql/src/main/java/org/apache/druid/sql/avatica/AbstractDruidJdbcStatement.java
@@ -77,7 +77,7 @@ public abstract class AbstractDruidJdbcStatement implements Closeable
params.add(createParameter(field, type));
}
return Meta.Signature.create(
- createColumnMetaData(prepareResult.getRowType()),
+ createColumnMetaData(prepareResult.getReturnedRowType()),
sql,
params,
Meta.CursorFactory.ARRAY,
diff --git a/sql/src/main/java/org/apache/druid/sql/avatica/DruidConnection.java b/sql/src/main/java/org/apache/druid/sql/avatica/DruidConnection.java
index 0c430b9216..addebc2f2d 100644
--- a/sql/src/main/java/org/apache/druid/sql/avatica/DruidConnection.java
+++ b/sql/src/main/java/org/apache/druid/sql/avatica/DruidConnection.java
@@ -78,7 +78,7 @@ public class DruidConnection
return connectionId;
}
- public DruidJdbcStatement createStatement(SqlStatementFactory sqlLifecycleFactory)
+ public DruidJdbcStatement createStatement(SqlStatementFactory sqlStatementFactory)
{
final int statementId = statementCounter.incrementAndGet();
@@ -98,7 +98,7 @@ public class DruidConnection
connectionId,
statementId,
context,
- sqlLifecycleFactory
+ sqlStatementFactory
);
statements.put(statementId, statement);
@@ -108,7 +108,7 @@ public class DruidConnection
}
public DruidJdbcPreparedStatement createPreparedStatement(
- SqlStatementFactory sqlLifecycleFactory,
+ SqlStatementFactory sqlStatementFactory,
SqlQueryPlus sqlRequest,
final long maxRowCount)
{
@@ -126,7 +126,7 @@ public class DruidConnection
}
@SuppressWarnings("GuardedBy")
- final PreparedStatement statement = sqlLifecycleFactory.preparedStatement(
+ final PreparedStatement statement = sqlStatementFactory.preparedStatement(
sqlRequest.withContext(context)
);
final DruidJdbcPreparedStatement jdbcStmt = new DruidJdbcPreparedStatement(
diff --git a/sql/src/main/java/org/apache/druid/sql/avatica/DruidMeta.java b/sql/src/main/java/org/apache/druid/sql/avatica/DruidMeta.java
index 545e10901a..89239a0bee 100644
--- a/sql/src/main/java/org/apache/druid/sql/avatica/DruidMeta.java
+++ b/sql/src/main/java/org/apache/druid/sql/avatica/DruidMeta.java
@@ -21,7 +21,6 @@ package org.apache.druid.sql.avatica;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -38,9 +37,9 @@ import org.apache.calcite.avatica.QueryState;
import org.apache.calcite.avatica.remote.AvaticaRuntimeException;
import org.apache.calcite.avatica.remote.Service.ErrorResponse;
import org.apache.calcite.avatica.remote.TypedValue;
+import org.apache.druid.guice.LazySingleton;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.UOE;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.query.QueryContext;
@@ -50,13 +49,14 @@ import org.apache.druid.server.security.AuthenticatorMapper;
import org.apache.druid.server.security.ForbiddenException;
import org.apache.druid.sql.SqlQueryPlus;
import org.apache.druid.sql.SqlStatementFactory;
+import org.apache.druid.sql.SqlStatementFactoryFactory;
import org.apache.druid.sql.calcite.planner.Calcites;
import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.sql.calcite.run.NativeSqlEngine;
import org.joda.time.Interval;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -70,6 +70,7 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+@LazySingleton
public class DruidMeta extends MetaImpl
{
/**
@@ -105,7 +106,7 @@ public class DruidMeta extends MetaImpl
"user", "password"
);
- private final SqlStatementFactory sqlLifecycleFactory;
+ private final SqlStatementFactory sqlStatementFactory;
private final ScheduledExecutorService exec;
private final AvaticaServerConfig config;
private final List<Authenticator> authenticators;
@@ -124,25 +125,41 @@ public class DruidMeta extends MetaImpl
@Inject
public DruidMeta(
- final SqlStatementFactory sqlLifecycleFactory,
+ final NativeSqlEngine engine,
+ final SqlStatementFactoryFactory sqlStatementFactoryFactory,
final AvaticaServerConfig config,
final ErrorHandler errorHandler,
final Injector injector
)
+ {
+ this(
+ sqlStatementFactoryFactory.factorize(engine),
+ config,
+ errorHandler,
+ Executors.newSingleThreadScheduledExecutor(
+ new ThreadFactoryBuilder()
+ .setNameFormat("DruidMeta-ScheduledExecutor-%d")
+ .setDaemon(true)
+ .build()
+ ),
+ injector.getInstance(AuthenticatorMapper.class).getAuthenticatorChain()
+ );
+ }
+
+ public DruidMeta(
+ final SqlStatementFactory sqlStatementFactory,
+ final AvaticaServerConfig config,
+ final ErrorHandler errorHandler,
+ final ScheduledExecutorService exec,
+ final List<Authenticator> authenticators
+ )
{
super(null);
- this.sqlLifecycleFactory = Preconditions.checkNotNull(sqlLifecycleFactory, "sqlLifecycleFactory");
+ this.sqlStatementFactory = sqlStatementFactory;
this.config = config;
this.errorHandler = errorHandler;
- this.exec = Executors.newSingleThreadScheduledExecutor(
- new ThreadFactoryBuilder()
- .setNameFormat(StringUtils.format("DruidMeta@%s-ScheduledExecutor", Integer.toHexString(hashCode())))
- .setDaemon(true)
- .build()
- );
-
- final AuthenticatorMapper authenticatorMapper = injector.getInstance(AuthenticatorMapper.class);
- this.authenticators = authenticatorMapper.getAuthenticatorChain();
+ this.exec = exec;
+ this.authenticators = authenticators;
}
@Override
@@ -219,7 +236,7 @@ public class DruidMeta extends MetaImpl
public StatementHandle createStatement(final ConnectionHandle ch)
{
try {
- final DruidJdbcStatement druidStatement = getDruidConnection(ch.id).createStatement(sqlLifecycleFactory);
+ final DruidJdbcStatement druidStatement = getDruidConnection(ch.id).createStatement(sqlStatementFactory);
return new StatementHandle(ch.id, druidStatement.getStatementId(), null);
}
catch (NoSuchConnectionException e) {
@@ -251,7 +268,7 @@ public class DruidMeta extends MetaImpl
doAuthenticate(druidConnection)
);
DruidJdbcPreparedStatement stmt = getDruidConnection(ch.id).createPreparedStatement(
- sqlLifecycleFactory,
+ sqlStatementFactory,
sqlReq,
maxRowCount);
stmt.prepare();
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/external/ExternalTableScanRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/external/ExternalTableScanRule.java
index 5e74ca02b9..fc3d8098ed 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/external/ExternalTableScanRule.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/external/ExternalTableScanRule.java
@@ -23,7 +23,7 @@ import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.druid.sql.calcite.planner.PlannerContext;
import org.apache.druid.sql.calcite.rel.DruidQueryRel;
-import org.apache.druid.sql.calcite.run.QueryFeature;
+import org.apache.druid.sql.calcite.run.EngineFeature;
/**
* Rule that converts an {@link ExternalTableScan} to a call to {@link DruidQueryRel#scanExternal}.
@@ -43,10 +43,15 @@ public class ExternalTableScanRule extends RelOptRule
@Override
public boolean matches(RelOptRuleCall call)
{
- if (plannerContext.getQueryMaker().feature(QueryFeature.CAN_READ_EXTERNAL_DATA)) {
+ if (plannerContext.engineHasFeature(EngineFeature.READ_EXTERNAL_DATA)) {
return super.matches(call);
} else {
- plannerContext.setPlanningError("SQL query requires scanning external datasources that is not suported.");
+ plannerContext.setPlanningError(
+ "Cannot use '%s' with SQL engine '%s'.",
+ ExternalOperatorConversion.FUNCTION_NAME,
+ plannerContext.getEngine().name()
+ );
+
return false;
}
}
@@ -54,6 +59,11 @@ public class ExternalTableScanRule extends RelOptRule
@Override
public void onMatch(final RelOptRuleCall call)
{
+ if (!plannerContext.engineHasFeature(EngineFeature.READ_EXTERNAL_DATA)) {
+ // Not called because "matches" returns false.
+ throw new UnsupportedOperationException();
+ }
+
final ExternalTableScan scan = call.rel(0);
call.transformTo(DruidQueryRel.scanExternal(scan, plannerContext));
}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModule.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModule.java
index 7e67ec5020..f911024bb4 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModule.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModule.java
@@ -20,40 +20,21 @@
package org.apache.druid.sql.calcite.planner;
import com.google.inject.Binder;
-import com.google.inject.Key;
import com.google.inject.Module;
import com.google.inject.multibindings.Multibinder;
import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.guice.LazySingleton;
-import org.apache.druid.guice.PolyBind;
import org.apache.druid.sql.calcite.rule.ExtensionCalciteRuleProvider;
-import org.apache.druid.sql.calcite.run.NativeQueryMakerFactory;
-import org.apache.druid.sql.calcite.run.QueryMakerFactory;
/**
* The module responsible for provide bindings for the Calcite Planner.
*/
public class CalcitePlannerModule implements Module
{
- public static final String PROPERTY_SQL_EXECUTOR_TYPE = "druid.sql.executor.type";
-
@Override
public void configure(Binder binder)
{
JsonConfigProvider.bind(binder, "druid.sql.planner", PlannerConfig.class);
-
- PolyBind.optionBinder(binder, Key.get(QueryMakerFactory.class))
- .addBinding(NativeQueryMakerFactory.TYPE)
- .to(NativeQueryMakerFactory.class)
- .in(LazySingleton.class);
-
- PolyBind.createChoiceWithDefault(
- binder,
- PROPERTY_SQL_EXECUTOR_TYPE,
- Key.get(QueryMakerFactory.class),
- NativeQueryMakerFactory.TYPE
- );
-
binder.bind(PlannerFactory.class).in(LazySingleton.class);
binder.bind(DruidOperatorTable.class).in(LazySingleton.class);
Multibinder.newSetBinder(binder, ExtensionCalciteRuleProvider.class);
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/Calcites.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/Calcites.java
index c81843b210..aa95fee287 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/Calcites.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/Calcites.java
@@ -58,7 +58,6 @@ import org.joda.time.format.DateTimeFormatterBuilder;
import org.joda.time.format.ISODateTimeFormat;
import javax.annotation.Nullable;
-
import java.math.BigDecimal;
import java.nio.charset.Charset;
import java.sql.Date;
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java
index cd51800e46..430bf24a35 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
@@ -37,15 +38,19 @@ import org.apache.calcite.jdbc.CalciteSchema;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.RelVisitor;
import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.logical.LogicalSort;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.schema.ScannableTable;
import org.apache.calcite.sql.SqlExplain;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlInsert;
@@ -81,13 +86,14 @@ import org.apache.druid.sql.calcite.rel.DruidConvention;
import org.apache.druid.sql.calcite.rel.DruidQuery;
import org.apache.druid.sql.calcite.rel.DruidRel;
import org.apache.druid.sql.calcite.rel.DruidUnionRel;
+import org.apache.druid.sql.calcite.run.EngineFeature;
import org.apache.druid.sql.calcite.run.QueryMaker;
-import org.apache.druid.sql.calcite.run.QueryMakerFactory;
+import org.apache.druid.sql.calcite.run.SqlEngine;
+import org.apache.druid.sql.calcite.table.DruidTable;
import org.apache.druid.utils.Throwables;
import org.joda.time.DateTimeZone;
import javax.annotation.Nullable;
-
import java.io.Closeable;
import java.util.ArrayList;
import java.util.HashSet;
@@ -127,7 +133,7 @@ public class DruidPlanner implements Closeable
private final FrameworkConfig frameworkConfig;
private final CalcitePlanner planner;
private final PlannerContext plannerContext;
- private final QueryMakerFactory queryMakerFactory;
+ private final SqlEngine engine;
private State state = State.START;
private ParsedNodes parsed;
private SqlNode validatedQueryNode;
@@ -140,13 +146,13 @@ public class DruidPlanner implements Closeable
DruidPlanner(
final FrameworkConfig frameworkConfig,
final PlannerContext plannerContext,
- final QueryMakerFactory queryMakerFactory
+ final SqlEngine engine
)
{
this.frameworkConfig = frameworkConfig;
this.planner = new CalcitePlanner(frameworkConfig);
this.plannerContext = plannerContext;
- this.queryMakerFactory = queryMakerFactory;
+ this.engine = engine;
}
/**
@@ -158,9 +164,22 @@ public class DruidPlanner implements Closeable
public void validate() throws SqlParseException, ValidationException
{
Preconditions.checkState(state == State.START);
+
+ // Validate query context.
+ engine.validateContext(plannerContext.getQueryContext());
+
+ // Parse the query string.
SqlNode root = planner.parse(plannerContext.getSql());
parsed = ParsedNodes.create(root, plannerContext.getTimeZone());
+ if (parsed.isSelect() && !plannerContext.engineHasFeature(EngineFeature.CAN_SELECT)) {
+ throw new ValidationException(StringUtils.format("Cannot execute SELECT with SQL engine '%s'.", engine.name()));
+ } else if (parsed.isInsert() && !plannerContext.engineHasFeature(EngineFeature.CAN_INSERT)) {
+ throw new ValidationException(StringUtils.format("Cannot execute INSERT with SQL engine '%s'.", engine.name()));
+ } else if (parsed.isReplace() && !plannerContext.engineHasFeature(EngineFeature.CAN_REPLACE)) {
+ throw new ValidationException(StringUtils.format("Cannot execute REPLACE with SQL engine '%s'.", engine.name()));
+ }
+
try {
if (parsed.getIngestionGranularity() != null) {
plannerContext.getQueryContext().addSystemParam(
@@ -205,11 +224,17 @@ public class DruidPlanner implements Closeable
resourceActions = new HashSet<>(resourceCollectorShuttle.getResourceActions());
- if (parsed.getInsertOrReplace() != null) {
+ if (parsed.isInsert() || parsed.isReplace()) {
// Check if CTX_SQL_OUTER_LIMIT is specified and fail the query if it is. CTX_SQL_OUTER_LIMIT being provided causes
// the number of rows inserted to be limited which is likely to be confusing and unintended.
if (plannerContext.getQueryContext().get(PlannerContext.CTX_SQL_OUTER_LIMIT) != null) {
- throw new ValidationException(PlannerContext.CTX_SQL_OUTER_LIMIT + " cannot be provided on INSERT or REPLACE queries.");
+ throw new ValidationException(
+ StringUtils.format(
+ "%s cannot be provided with %s.",
+ PlannerContext.CTX_SQL_OUTER_LIMIT,
+ parsed.getInsertOrReplace().getOperator().getName()
+ )
+ );
}
final String targetDataSource = validateAndGetDataSourceForIngest(parsed.getInsertOrReplace());
resourceActions.add(new ResourceAction(new Resource(targetDataSource, ResourceType.DATASOURCE), Action.WRITE));
@@ -232,17 +257,17 @@ public class DruidPlanner implements Closeable
* {@link org.apache.druid.sql.calcite.view.DruidViewMacro} prepares
* a view while having no information about the user of that view.
*/
- public PrepareResult prepare() throws ValidationException
+ public PrepareResult prepare()
{
Preconditions.checkState(state == State.VALIDATED);
rootQueryRel = planner.rel(validatedQueryNode);
- doPrepare(null);
+ doPrepare();
state = State.PREPARED;
return prepareResult;
}
- private void doPrepare(@Nullable QueryMaker queryMaker) throws ValidationException
+ private void doPrepare()
{
final RelDataTypeFactory typeFactory = rootQueryRel.rel.getCluster().getTypeFactory();
final SqlValidator validator = planner.getValidator();
@@ -251,14 +276,14 @@ public class DruidPlanner implements Closeable
if (parsed.getExplainNode() != null) {
returnedRowType = getExplainStructType(typeFactory);
+ } else if (parsed.isSelect()) {
+ returnedRowType = engine.resultTypeForSelect(typeFactory, rootQueryRel.validatedRowType);
} else {
- if (queryMaker == null) {
- queryMaker = buildQueryMaker(rootQueryRel, parsed.getInsertOrReplace());
- }
- returnedRowType = queryMaker.getResultType();
+ assert parsed.insertOrReplace != null;
+ returnedRowType = engine.resultTypeForInsert(typeFactory, rootQueryRel.validatedRowType);
}
- prepareResult = new PrepareResult(returnedRowType, parameterTypes);
+ prepareResult = new PrepareResult(rootQueryRel.validatedRowType, returnedRowType, parameterTypes);
}
/**
@@ -266,6 +291,7 @@ public class DruidPlanner implements Closeable
* step within the planner's state machine.
*
* @param authorizer a function from resource actions to a {@link Access} result.
+ *
* @return the return value from the authorizer
*/
public Access authorize(Function<Set<ResourceAction>, Access> authorizer, boolean authorizeContextParams)
@@ -318,31 +344,47 @@ public class DruidPlanner implements Closeable
rootQueryRel = planner.rel(validatedQueryNode);
}
+ final Set<RelOptTable> bindableTables = getBindableTables(rootQueryRel.rel);
+
// the planner's type factory is not available until after parsing
this.rexBuilder = new RexBuilder(planner.getTypeFactory());
state = State.PLANNED;
try {
- return planWithDruidConvention(rootQueryRel, parsed.getExplainNode(), parsed.getInsertOrReplace());
+ if (!bindableTables.isEmpty()) {
+ // Consider BINDABLE convention when necessary. Used for metadata tables.
+
+ if (parsed.isInsert() || parsed.isReplace()) {
+ // Throws ValidationException if the target table is itself bindable.
+ validateAndGetDataSourceForIngest(parsed.getInsertOrReplace());
+ }
+
+ if (!plannerContext.engineHasFeature(EngineFeature.ALLOW_BINDABLE_PLAN)) {
+ throw new ValidationException(
+ StringUtils.format(
+ "Cannot query table%s [%s] with SQL engine '%s'.",
+ bindableTables.size() != 1 ? "s" : "",
+ bindableTables.stream()
+ .map(table -> Joiner.on(".").join(table.getQualifiedName()))
+ .collect(Collectors.joining(", ")),
+ engine.name()
+ )
+ );
+ }
+
+ return planWithBindableConvention(rootQueryRel, parsed.getExplainNode());
+ } else {
+ // DRUID convention is used whenever there are no tables that require BINDABLE.
+ return planWithDruidConvention(rootQueryRel, parsed.getExplainNode(), parsed.getInsertOrReplace());
+ }
}
catch (Exception e) {
Throwable cannotPlanException = Throwables.getCauseOfType(e, RelOptPlanner.CannotPlanException.class);
if (null == cannotPlanException) {
- // Not a CannotPlanException, rethrow without trying with bindable
+ // Not a CannotPlanException, rethrow without logging.
throw e;
}
- // If there isn't any ingestion clause, then we should try again with BINDABLE convention. And return without
- // any error, if it is plannable by the bindable convention
- if (parsed.getInsertOrReplace() == null) {
- // Try again with BINDABLE convention. Used for querying Values and metadata tables.
- try {
- return planWithBindableConvention(rootQueryRel, parsed.getExplainNode());
- }
- catch (Exception e2) {
- e.addSuppressed(e2);
- }
- }
Logger logger = log;
if (!plannerContext.getQueryContext().isDebug()) {
logger = log.noStackTrace();
@@ -382,7 +424,7 @@ public class DruidPlanner implements Closeable
final QueryMaker queryMaker = buildQueryMaker(possiblyLimitedRoot, insertOrReplace);
plannerContext.setQueryMaker(queryMaker);
if (prepareResult == null) {
- doPrepare(queryMaker);
+ doPrepare();
}
// Fall-back dynamic parameter substitution using {@link RelParameterizerShuttle}
@@ -408,8 +450,19 @@ public class DruidPlanner implements Closeable
if (explain != null) {
return planExplanation(druidRel, explain, true);
} else {
- final Supplier<Sequence<Object[]>> resultsSupplier = () -> {
+ // Compute row type.
+ final RelDataTypeFactory typeFactory = rootQueryRel.rel.getCluster().getTypeFactory();
+ final RelDataType rowType;
+
+ if (parsed.isSelect()) {
+ rowType = engine.resultTypeForSelect(typeFactory, rootQueryRel.validatedRowType);
+ } else {
+ assert parsed.insertOrReplace != null;
+ rowType = engine.resultTypeForInsert(typeFactory, rootQueryRel.validatedRowType);
+ }
+ // Start the query.
+ final Supplier<Sequence<Object[]>> resultsSupplier = () -> {
// sanity check
final Set<ResourceAction> readResourceActions =
plannerContext.getResourceActions()
@@ -428,7 +481,7 @@ public class DruidPlanner implements Closeable
return druidRel.runQuery();
};
- return new PlannerResult(resultsSupplier, queryMaker.getResultType());
+ return new PlannerResult(resultsSupplier, rowType);
}
}
@@ -449,6 +502,10 @@ public class DruidPlanner implements Closeable
@Nullable final SqlExplain explain
)
{
+ if (prepareResult == null) {
+ doPrepare();
+ }
+
BindableRel bindableRel = (BindableRel) planner.transform(
CalciteRulesManager.BINDABLE_CONVENTION_RULES,
planner.getEmptyTraitSet().replace(BindableConvention.INSTANCE).plus(root.collation),
@@ -641,6 +698,7 @@ public class DruidPlanner implements Closeable
* the web console, allowing it to apply a limit to queries without rewriting the original SQL.
*
* @param root root node
+ *
* @return root node wrapped with a limiting logical sort if a limit is specified in the query context.
*/
@Nullable
@@ -691,9 +749,9 @@ public class DruidPlanner implements Closeable
if (insertOrReplace != null) {
final String targetDataSource = validateAndGetDataSourceForIngest(insertOrReplace);
validateColumnsForIngestion(rootQueryRel);
- return queryMakerFactory.buildForInsert(targetDataSource, rootQueryRel, plannerContext);
+ return engine.buildQueryMakerForInsert(targetDataSource, rootQueryRel, plannerContext);
} else {
- return queryMakerFactory.buildForSelect(rootQueryRel, plannerContext);
+ return engine.buildQueryMakerForSelect(rootQueryRel, plannerContext);
}
}
@@ -741,7 +799,12 @@ public class DruidPlanner implements Closeable
dataSource = tableIdentifier.names.get(1);
} else {
throw new ValidationException(
- StringUtils.format("Cannot %s into [%s] because it is not a Druid datasource.", operatorName, tableIdentifier)
+ StringUtils.format(
+ "Cannot %s into [%s] because it is not a Druid datasource (schema = %s).",
+ operatorName,
+ tableIdentifier,
+ defaultSchemaName
+ )
);
}
}
@@ -773,13 +836,39 @@ public class DruidPlanner implements Closeable
errorMessage = exception.getMessage();
}
if (null == errorMessage) {
- errorMessage = "Please check broker logs for more details";
+ errorMessage = "Please check Broker logs for more details.";
} else {
// Re-phrase since planning errors are more like hints
errorMessage = "Possible error: " + errorMessage;
}
// Finally, add the query itself to error message that user will get.
- return StringUtils.format("Cannot build plan for query: %s. %s", plannerContext.getSql(), errorMessage);
+ return StringUtils.format("Cannot build plan for query. %s", errorMessage);
+ }
+
+ private static Set<RelOptTable> getBindableTables(final RelNode relNode)
+ {
+ class HasBindableVisitor extends RelVisitor
+ {
+ private final Set<RelOptTable> found = new HashSet<>();
+
+ @Override
+ public void visit(RelNode node, int ordinal, RelNode parent)
+ {
+ if (node instanceof TableScan) {
+ RelOptTable table = node.getTable();
+ if (table.unwrap(ScannableTable.class) != null && table.unwrap(DruidTable.class) == null) {
+ found.add(table);
+ return;
+ }
+ }
+
+ super.visit(node, ordinal, parent);
+ }
+ }
+
+ final HasBindableVisitor visitor = new HasBindableVisitor();
+ visitor.go(relNode);
+ return visitor.found;
}
private static class EnumeratorIterator<T> implements Iterator<T>
@@ -924,6 +1013,21 @@ public class DruidPlanner implements Closeable
return explain;
}
+ public boolean isSelect()
+ {
+ return insertOrReplace == null;
+ }
+
+ public boolean isInsert()
+ {
+ return insertOrReplace != null && !isReplace();
+ }
+
+ public boolean isReplace()
+ {
+ return insertOrReplace instanceof DruidSqlReplace;
+ }
+
@Nullable
public SqlInsert getInsertOrReplace()
{
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java
index 1aae09c4ae..616823b1a6 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java
@@ -39,7 +39,9 @@ import org.apache.druid.server.security.Access;
import org.apache.druid.server.security.AuthenticationResult;
import org.apache.druid.server.security.ResourceAction;
import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry;
+import org.apache.druid.sql.calcite.run.EngineFeature;
import org.apache.druid.sql.calcite.run.QueryMaker;
+import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
@@ -79,6 +81,7 @@ public class PlannerContext
private final PlannerConfig plannerConfig;
private final DateTime localNow;
private final DruidSchemaCatalog rootSchema;
+ private final SqlEngine engine;
private final QueryContext queryContext;
private final String sqlQueryId;
private final boolean stringifyArrays;
@@ -106,6 +109,7 @@ public class PlannerContext
final DateTime localNow,
final boolean stringifyArrays,
final DruidSchemaCatalog rootSchema,
+ final SqlEngine engine,
final QueryContext queryContext
)
{
@@ -115,6 +119,7 @@ public class PlannerContext
this.jsonMapper = jsonMapper;
this.plannerConfig = Preconditions.checkNotNull(plannerConfig, "plannerConfig");
this.rootSchema = rootSchema;
+ this.engine = engine;
this.queryContext = queryContext;
this.localNow = Preconditions.checkNotNull(localNow, "localNow");
this.stringifyArrays = stringifyArrays;
@@ -134,6 +139,7 @@ public class PlannerContext
final ObjectMapper jsonMapper,
final PlannerConfig plannerConfig,
final DruidSchemaCatalog rootSchema,
+ final SqlEngine engine,
final QueryContext queryContext
)
{
@@ -172,6 +178,7 @@ public class PlannerContext
utcNow.withZone(timeZone),
stringifyArrays,
rootSchema,
+ engine,
queryContext
);
}
@@ -383,6 +390,16 @@ public class PlannerContext
this.queryMaker = Preconditions.checkNotNull(queryMaker, "queryMaker");
}
+ public SqlEngine getEngine()
+ {
+ return engine;
+ }
+
+ public boolean engineHasFeature(final EngineFeature feature)
+ {
+ return engine.feature(feature, this);
+ }
+
public QueryMaker getQueryMaker()
{
return Preconditions.checkNotNull(queryMaker, "QueryMaker not available");
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java
index 7b6f507947..d8852a87a4 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java
@@ -45,11 +45,10 @@ import org.apache.druid.server.security.AuthorizerMapper;
import org.apache.druid.server.security.NoopEscalator;
import org.apache.druid.sql.calcite.parser.DruidSqlParserImplFactory;
import org.apache.druid.sql.calcite.planner.convertlet.DruidConvertletTable;
-import org.apache.druid.sql.calcite.run.QueryMakerFactory;
+import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.schema.DruidSchemaName;
-import java.util.Map;
import java.util.Properties;
public class PlannerFactory
@@ -65,7 +64,6 @@ public class PlannerFactory
.build();
private final DruidSchemaCatalog rootSchema;
- private final QueryMakerFactory queryMakerFactory;
private final DruidOperatorTable operatorTable;
private final ExprMacroTable macroTable;
private final PlannerConfig plannerConfig;
@@ -77,7 +75,6 @@ public class PlannerFactory
@Inject
public PlannerFactory(
final DruidSchemaCatalog rootSchema,
- final QueryMakerFactory queryMakerFactory,
final DruidOperatorTable operatorTable,
final ExprMacroTable macroTable,
final PlannerConfig plannerConfig,
@@ -88,7 +85,6 @@ public class PlannerFactory
)
{
this.rootSchema = rootSchema;
- this.queryMakerFactory = queryMakerFactory;
this.operatorTable = operatorTable;
this.macroTable = macroTable;
this.plannerConfig = plannerConfig;
@@ -101,7 +97,7 @@ public class PlannerFactory
/**
* Create a Druid query planner from an initial query context
*/
- public DruidPlanner createPlanner(final String sql, final QueryContext queryContext)
+ public DruidPlanner createPlanner(final SqlEngine engine, final String sql, final QueryContext queryContext)
{
final PlannerContext context = PlannerContext.create(
sql,
@@ -110,10 +106,11 @@ public class PlannerFactory
jsonMapper,
plannerConfig,
rootSchema,
+ engine,
queryContext
);
- return new DruidPlanner(buildFrameworkConfig(context), context, queryMakerFactory);
+ return new DruidPlanner(buildFrameworkConfig(context), context, engine);
}
/**
@@ -121,9 +118,9 @@ public class PlannerFactory
* and ready to go authorization result.
*/
@VisibleForTesting
- public DruidPlanner createPlannerForTesting(final Map<String, Object> queryContext, String query)
+ public DruidPlanner createPlannerForTesting(final SqlEngine engine, final String sql, final QueryContext queryContext)
{
- final DruidPlanner thePlanner = createPlanner(query, new QueryContext(queryContext));
+ final DruidPlanner thePlanner = createPlanner(engine, sql, queryContext);
thePlanner.getPlannerContext()
.setAuthenticationResult(NoopEscalator.getInstance().createEscalatedAuthenticationResult());
try {
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerResult.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerResult.java
index 666af5ff15..c571c42ab0 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerResult.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerResult.java
@@ -56,6 +56,9 @@ public class PlannerResult
return resultsSupplier.get();
}
+ /**
+ * Row type returned to the end user. Equivalent to {@link PrepareResult#getReturnedRowType()}.
+ */
public RelDataType rowType()
{
return rowType;
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PrepareResult.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PrepareResult.java
index c6660d8adc..6c92016428 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PrepareResult.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PrepareResult.java
@@ -27,20 +27,46 @@ import org.apache.calcite.rel.type.RelDataType;
*/
public class PrepareResult
{
- private final RelDataType rowType;
+ private final RelDataType validatedRowType;
+ private final RelDataType returnedRowType;
private final RelDataType parameterRowType;
- public PrepareResult(final RelDataType rowType, final RelDataType parameterRowType)
+ public PrepareResult(
+ final RelDataType validatedRowType,
+ final RelDataType returnedRowType,
+ final RelDataType parameterRowType
+ )
{
- this.rowType = rowType;
+ this.validatedRowType = validatedRowType;
+ this.returnedRowType = returnedRowType;
this.parameterRowType = parameterRowType;
}
- public RelDataType getRowType()
+ /**
+ * Row type from {@link org.apache.calcite.rel.RelRoot#validatedRowType} prepared by {@link DruidPlanner#prepare()}.
+ * Corresponds to the SELECT portion of a SQL statement. For SELECT, this is the row type of the SELECT itself.
+ * For EXPLAIN PLAN FOR SELECT, INSERT ... SELECT, or REPLACE ... SELECT, this is the row type of the
+ * embedded SELECT.
+ */
+ public RelDataType getValidatedRowType()
{
- return rowType;
+ return validatedRowType;
}
+ /**
+ * Row type for the result that the end user will receive. Different from {@link #getValidatedRowType()} in
+ * cases like EXPLAIN (where the user gets an explanation, not the query results) and other non-SELECT
+ * statements.
+ */
+ public RelDataType getReturnedRowType()
+ {
+ return returnedRowType;
+ }
+
+ /**
+ * Row type from {@link org.apache.calcite.sql.validate.SqlValidator#getParameterRowType} containing the
+ * name and type of each parameter.
+ */
public RelDataType getParameterRowType()
{
return parameterRowType;
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
index 075401eff1..9c6616cbd7 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java
@@ -19,6 +19,7 @@
package org.apache.druid.sql.calcite.rel;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -48,6 +49,7 @@ import org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.query.DataSource;
import org.apache.druid.query.JoinDataSource;
import org.apache.druid.query.Query;
+import org.apache.druid.query.QueryContext;
import org.apache.druid.query.QueryDataSource;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.LongMaxAggregatorFactory;
@@ -72,6 +74,7 @@ import org.apache.druid.query.topn.TopNMetricSpec;
import org.apache.druid.query.topn.TopNQuery;
import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
@@ -86,8 +89,7 @@ import org.apache.druid.sql.calcite.planner.Calcites;
import org.apache.druid.sql.calcite.planner.OffsetLimit;
import org.apache.druid.sql.calcite.planner.PlannerContext;
import org.apache.druid.sql.calcite.rule.GroupByRules;
-import org.apache.druid.sql.calcite.run.QueryFeature;
-import org.apache.druid.sql.calcite.run.QueryFeatureInspector;
+import org.apache.druid.sql.calcite.run.EngineFeature;
import org.apache.druid.sql.calcite.table.RowSignatures;
import javax.annotation.Nonnull;
@@ -111,6 +113,11 @@ import java.util.stream.Collectors;
*/
public class DruidQuery
{
+ /**
+ * Native query context key that is set when {@link EngineFeature#SCAN_NEEDS_SIGNATURE}.
+ */
+ public static final String CTX_SCAN_SIGNATURE = "scanSignature";
+
private final DataSource dataSource;
private final PlannerContext plannerContext;
@@ -130,6 +137,7 @@ public class DruidQuery
private final RowSignature outputRowSignature;
private final RelDataType outputRowType;
private final VirtualColumnRegistry virtualColumnRegistry;
+ private final RowSignature sourceRowSignature;
private DruidQuery(
final DataSource dataSource,
@@ -149,10 +157,12 @@ public class DruidQuery
this.selectProjection = selectProjection;
this.grouping = grouping;
this.sorting = sorting;
+ this.sourceRowSignature = sourceRowSignature;
+
this.outputRowSignature = computeOutputRowSignature(sourceRowSignature, selectProjection, grouping, sorting);
this.outputRowType = Preconditions.checkNotNull(outputRowType, "outputRowType");
this.virtualColumnRegistry = Preconditions.checkNotNull(virtualColumnRegistry, "virtualColumnRegistry");
- this.query = computeQuery(plannerContext.getQueryMaker());
+ this.query = computeQuery();
}
public static DruidQuery fromPartialQuery(
@@ -796,40 +806,40 @@ public class DruidQuery
*
* @return Druid query
*/
- private Query computeQuery(final QueryFeatureInspector queryFeatureInspector)
+ private Query computeQuery()
{
if (dataSource instanceof QueryDataSource) {
// If there is a subquery, then we prefer the outer query to be a groupBy if possible, since this potentially
// enables more efficient execution. (The groupBy query toolchest can handle some subqueries by itself, without
// requiring the Broker to inline results.)
- final GroupByQuery outerQuery = toGroupByQuery(queryFeatureInspector);
+ final GroupByQuery outerQuery = toGroupByQuery();
if (outerQuery != null) {
return outerQuery;
}
}
- final TimeBoundaryQuery timeBoundaryQuery = toTimeBoundaryQuery(queryFeatureInspector);
+ final TimeBoundaryQuery timeBoundaryQuery = toTimeBoundaryQuery();
if (timeBoundaryQuery != null) {
return timeBoundaryQuery;
}
- final TimeseriesQuery tsQuery = toTimeseriesQuery(queryFeatureInspector);
+ final TimeseriesQuery tsQuery = toTimeseriesQuery();
if (tsQuery != null) {
return tsQuery;
}
- final TopNQuery topNQuery = toTopNQuery(queryFeatureInspector);
+ final TopNQuery topNQuery = toTopNQuery();
if (topNQuery != null) {
return topNQuery;
}
- final GroupByQuery groupByQuery = toGroupByQuery(queryFeatureInspector);
+ final GroupByQuery groupByQuery = toGroupByQuery();
if (groupByQuery != null) {
return groupByQuery;
}
- final ScanQuery scanQuery = toScanQuery(queryFeatureInspector);
+ final ScanQuery scanQuery = toScanQuery();
if (scanQuery != null) {
return scanQuery;
}
@@ -843,9 +853,9 @@ public class DruidQuery
* @return a TimeBoundaryQuery if possible. null if it is not possible to construct one.
*/
@Nullable
- private TimeBoundaryQuery toTimeBoundaryQuery(QueryFeatureInspector queryFeatureInspector)
+ private TimeBoundaryQuery toTimeBoundaryQuery()
{
- if (!queryFeatureInspector.feature(QueryFeature.CAN_RUN_TIME_BOUNDARY)
+ if (!plannerContext.engineHasFeature(EngineFeature.TIME_BOUNDARY_QUERY)
|| grouping == null
|| grouping.getSubtotals().hasEffect(grouping.getDimensionSpecs())
|| grouping.getHavingFilter() != null
@@ -907,9 +917,9 @@ public class DruidQuery
* @return query
*/
@Nullable
- private TimeseriesQuery toTimeseriesQuery(final QueryFeatureInspector queryFeatureInspector)
+ private TimeseriesQuery toTimeseriesQuery()
{
- if (!queryFeatureInspector.feature(QueryFeature.CAN_RUN_TIMESERIES)
+ if (!plannerContext.engineHasFeature(EngineFeature.TIMESERIES_QUERY)
|| grouping == null
|| grouping.getSubtotals().hasEffect(grouping.getDimensionSpecs())
|| grouping.getHavingFilter() != null) {
@@ -1019,10 +1029,10 @@ public class DruidQuery
* @return query or null
*/
@Nullable
- private TopNQuery toTopNQuery(final QueryFeatureInspector queryFeatureInspector)
+ private TopNQuery toTopNQuery()
{
// Must be allowed by the QueryMaker.
- if (!queryFeatureInspector.feature(QueryFeature.CAN_RUN_TOPN)) {
+ if (!plannerContext.engineHasFeature(EngineFeature.TOPN_QUERY)) {
return null;
}
@@ -1114,7 +1124,7 @@ public class DruidQuery
* @return query or null
*/
@Nullable
- private GroupByQuery toGroupByQuery(final QueryFeatureInspector queryFeatureInspector)
+ private GroupByQuery toGroupByQuery()
{
if (grouping == null) {
return null;
@@ -1227,7 +1237,7 @@ public class DruidQuery
* @return query or null
*/
@Nullable
- private ScanQuery toScanQuery(final QueryFeatureInspector queryFeatureInspector)
+ private ScanQuery toScanQuery()
{
if (grouping != null) {
// Scan cannot GROUP BY.
@@ -1278,7 +1288,7 @@ public class DruidQuery
orderByColumns = Collections.emptyList();
}
- if (!queryFeatureInspector.feature(QueryFeature.SCAN_CAN_ORDER_BY_NON_TIME) && !orderByColumns.isEmpty()) {
+ if (!plannerContext.engineHasFeature(EngineFeature.SCAN_ORDER_BY_NON_TIME) && !orderByColumns.isEmpty()) {
if (orderByColumns.size() > 1 || !ColumnHolder.TIME_COLUMN_NAME.equals(orderByColumns.get(0).getColumnName())) {
// Cannot handle this ordering.
// Scan cannot ORDER BY non-time columns.
@@ -1303,10 +1313,13 @@ public class DruidQuery
final SortedSet<String> scanColumns = new TreeSet<>(outputRowSignature.getColumnNames());
orderByColumns.forEach(column -> scanColumns.add(column.getColumnName()));
+ final VirtualColumns virtualColumns = getVirtualColumns(true);
+ final ImmutableList<String> scanColumnsList = ImmutableList.copyOf(scanColumns);
+
return new ScanQuery(
newDataSource,
filtration.getQuerySegmentSpec(),
- getVirtualColumns(true),
+ virtualColumns,
ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST,
0,
scanOffset,
@@ -1314,9 +1327,56 @@ public class DruidQuery
null,
orderByColumns,
filtration.getDimFilter(),
- ImmutableList.copyOf(scanColumns),
+ scanColumnsList,
false,
- ImmutableSortedMap.copyOf(plannerContext.getQueryContext().getMergedParams())
+ withScanSignatureIfNeeded(
+ virtualColumns,
+ scanColumnsList,
+ plannerContext.getQueryContext()
+ ).getMergedParams()
);
}
+
+ /**
+ * Returns a copy of "queryContext" with {@link #CTX_SCAN_SIGNATURE} added if the execution context has the
+ * {@link EngineFeature#SCAN_NEEDS_SIGNATURE} feature.
+ */
+ private QueryContext withScanSignatureIfNeeded(
+ final VirtualColumns virtualColumns,
+ final List<String> scanColumns,
+ final QueryContext queryContext
+ )
+ {
+ if (plannerContext.engineHasFeature(EngineFeature.SCAN_NEEDS_SIGNATURE)) {
+ // Compute the signature of the columns that we are selecting.
+ final RowSignature.Builder scanSignatureBuilder = RowSignature.builder();
+
+ for (final String columnName : scanColumns) {
+ final ColumnCapabilities capabilities =
+ virtualColumns.getColumnCapabilitiesWithFallback(sourceRowSignature, columnName);
+
+ if (capabilities == null) {
+ // No type for this column. This is a planner bug.
+ throw new ISE("No type for column [%s]", columnName);
+ }
+
+ scanSignatureBuilder.add(columnName, capabilities.toColumnType());
+ }
+
+ final RowSignature signature = scanSignatureBuilder.build();
+
+ try {
+ queryContext.addSystemParam(
+ CTX_SCAN_SIGNATURE,
+ plannerContext.getJsonMapper().writeValueAsString(signature)
+ );
+ return queryContext;
+ }
+ catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ return queryContext;
+ }
+ }
}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/QueryFeature.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/EngineFeature.java
similarity index 62%
rename from sql/src/main/java/org/apache/druid/sql/calcite/run/QueryFeature.java
rename to sql/src/main/java/org/apache/druid/sql/calcite/run/EngineFeature.java
index 363837435d..e8a9bd969d 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/run/QueryFeature.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/run/EngineFeature.java
@@ -20,35 +20,63 @@
package org.apache.druid.sql.calcite.run;
import org.apache.druid.sql.calcite.external.ExternalDataSource;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
/**
- * Arguments to {@link QueryFeatureInspector#feature(QueryFeature)}.
+ * Arguments to {@link SqlEngine#feature(EngineFeature, PlannerContext)}.
*/
-public enum QueryFeature
+public enum EngineFeature
{
+ /**
+ * Can execute SELECT statements.
+ */
+ CAN_SELECT,
+
+ /**
+ * Can execute INSERT statements.
+ */
+ CAN_INSERT,
+
+ /**
+ * Can execute REPLACE statements.
+ */
+ CAN_REPLACE,
+
/**
* Queries of type {@link org.apache.druid.query.timeseries.TimeseriesQuery} are usable.
*/
- CAN_RUN_TIMESERIES,
+ TIMESERIES_QUERY,
/**
* Queries of type {@link org.apache.druid.query.topn.TopNQuery} are usable.
*/
- CAN_RUN_TOPN,
+ TOPN_QUERY,
+
+ /**
+ * Queries of type {@link org.apache.druid.query.timeboundary.TimeBoundaryQuery} are usable.
+ */
+ TIME_BOUNDARY_QUERY,
/**
* Queries can use {@link ExternalDataSource}.
*/
- CAN_READ_EXTERNAL_DATA,
+ READ_EXTERNAL_DATA,
/**
* Scan queries can use {@link org.apache.druid.query.scan.ScanQuery#getOrderBys()} that are based on something
* other than the "__time" column.
*/
- SCAN_CAN_ORDER_BY_NON_TIME,
+ SCAN_ORDER_BY_NON_TIME,
/**
- * Queries of type {@link org.apache.druid.query.timeboundary.TimeBoundaryQuery} are usable.
+ * Scan queries must have {@link org.apache.druid.sql.calcite.rel.DruidQuery#CTX_SCAN_SIGNATURE} set in their
+ * query contexts.
+ */
+ SCAN_NEEDS_SIGNATURE,
+
+ /**
+ * Planner is permitted to use a {@link org.apache.calcite.runtime.Bindable} plan on local resources, instead
+ * of {@link QueryMaker}, for SELECT query implementation. Used for system tables and the like.
*/
- CAN_RUN_TIME_BOUNDARY
+ ALLOW_BINDABLE_PLAN
}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java
index 1bd6c8cb62..73a6f2cca4 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMaker.java
@@ -25,14 +25,12 @@ import com.google.common.collect.Iterables;
import com.google.common.primitives.Ints;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
-import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.runtime.Hook;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.NlsString;
import org.apache.calcite.util.Pair;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.DateTimes;
-import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.StringUtils;
@@ -42,7 +40,6 @@ import org.apache.druid.java.util.common.guava.Sequences;
import org.apache.druid.math.expr.Evals;
import org.apache.druid.query.InlineDataSource;
import org.apache.druid.query.Query;
-import org.apache.druid.query.QueryContexts;
import org.apache.druid.query.QueryToolChest;
import org.apache.druid.query.filter.BoundDimFilter;
import org.apache.druid.query.filter.DimFilter;
@@ -80,44 +77,18 @@ public class NativeQueryMaker implements QueryMaker
private final PlannerContext plannerContext;
private final ObjectMapper jsonMapper;
private final List<Pair<Integer, String>> fieldMapping;
- private final RelDataType resultType;
public NativeQueryMaker(
final QueryLifecycleFactory queryLifecycleFactory,
final PlannerContext plannerContext,
final ObjectMapper jsonMapper,
- final List<Pair<Integer, String>> fieldMapping,
- final RelDataType resultType
+ final List<Pair<Integer, String>> fieldMapping
)
{
this.queryLifecycleFactory = queryLifecycleFactory;
this.plannerContext = plannerContext;
this.jsonMapper = jsonMapper;
this.fieldMapping = fieldMapping;
- this.resultType = resultType;
- }
-
- @Override
- public RelDataType getResultType()
- {
- return resultType;
- }
-
- @Override
- public boolean feature(QueryFeature feature)
- {
- switch (feature) {
- case CAN_RUN_TIMESERIES:
- case CAN_RUN_TOPN:
- return true;
- case CAN_READ_EXTERNAL_DATA:
- case SCAN_CAN_ORDER_BY_NON_TIME:
- return false;
- case CAN_RUN_TIME_BOUNDARY:
- return QueryContexts.isTimeBoundaryPlanningEnabled(plannerContext.getQueryContext().getMergedParams());
- default:
- throw new IAE("Unrecognized feature: %s", feature);
- }
}
@Override
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMakerFactory.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMakerFactory.java
deleted file mode 100644
index 7553f46309..0000000000
--- a/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeQueryMakerFactory.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.sql.calcite.run;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.inject.Inject;
-import org.apache.calcite.rel.RelRoot;
-import org.apache.calcite.tools.ValidationException;
-import org.apache.druid.guice.LazySingleton;
-import org.apache.druid.server.QueryLifecycleFactory;
-import org.apache.druid.sql.calcite.planner.PlannerContext;
-
-@LazySingleton
-public class NativeQueryMakerFactory implements QueryMakerFactory
-{
- public static final String TYPE = "native";
-
- private final QueryLifecycleFactory queryLifecycleFactory;
- private final ObjectMapper jsonMapper;
-
- @Inject
- public NativeQueryMakerFactory(
- final QueryLifecycleFactory queryLifecycleFactory,
- final ObjectMapper jsonMapper
- )
- {
- this.queryLifecycleFactory = queryLifecycleFactory;
- this.jsonMapper = jsonMapper;
- }
-
- @Override
- public QueryMaker buildForSelect(final RelRoot relRoot, final PlannerContext plannerContext)
- {
- return new NativeQueryMaker(
- queryLifecycleFactory,
- plannerContext,
- jsonMapper,
- relRoot.fields,
- relRoot.validatedRowType
- );
- }
-
- @Override
- public QueryMaker buildForInsert(
- final String targetDataSource,
- final RelRoot relRoot,
- final PlannerContext plannerContext
- ) throws ValidationException
- {
- throw new ValidationException("Cannot execute INSERT queries.");
- }
-}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeSqlEngine.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeSqlEngine.java
new file mode 100644
index 0000000000..e207a4d3c2
--- /dev/null
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeSqlEngine.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.calcite.run;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Inject;
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.tools.ValidationException;
+import org.apache.druid.guice.LazySingleton;
+import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.query.QueryContext;
+import org.apache.druid.query.QueryContexts;
+import org.apache.druid.query.groupby.GroupByQuery;
+import org.apache.druid.query.timeboundary.TimeBoundaryQuery;
+import org.apache.druid.server.QueryLifecycleFactory;
+import org.apache.druid.sql.calcite.parser.DruidSqlInsert;
+import org.apache.druid.sql.calcite.parser.DruidSqlReplace;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.sql.calcite.rel.DruidQuery;
+
+import java.util.Set;
+
+@LazySingleton
+public class NativeSqlEngine implements SqlEngine
+{
+ public static final Set<String> SYSTEM_CONTEXT_PARAMETERS = ImmutableSet.of(
+ TimeBoundaryQuery.MAX_TIME_ARRAY_OUTPUT_NAME,
+ TimeBoundaryQuery.MIN_TIME_ARRAY_OUTPUT_NAME,
+ GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD,
+ GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_GRANULARITY,
+ GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_INDEX,
+ DruidQuery.CTX_SCAN_SIGNATURE,
+ DruidSqlInsert.SQL_INSERT_SEGMENT_GRANULARITY,
+ DruidSqlReplace.SQL_REPLACE_TIME_CHUNKS
+ );
+
+ private static final String NAME = "native";
+
+ private final QueryLifecycleFactory queryLifecycleFactory;
+ private final ObjectMapper jsonMapper;
+
+ @Inject
+ public NativeSqlEngine(
+ final QueryLifecycleFactory queryLifecycleFactory,
+ final ObjectMapper jsonMapper
+ )
+ {
+ this.queryLifecycleFactory = queryLifecycleFactory;
+ this.jsonMapper = jsonMapper;
+ }
+
+ @Override
+ public String name()
+ {
+ return NAME;
+ }
+
+ @Override
+ public void validateContext(QueryContext queryContext) throws ValidationException
+ {
+ SqlEngines.validateNoSpecialContextKeys(queryContext, SYSTEM_CONTEXT_PARAMETERS);
+ }
+
+ @Override
+ public RelDataType resultTypeForSelect(RelDataTypeFactory typeFactory, RelDataType validatedRowType)
+ {
+ return validatedRowType;
+ }
+
+ @Override
+ public RelDataType resultTypeForInsert(RelDataTypeFactory typeFactory, RelDataType validatedRowType)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean feature(EngineFeature feature, PlannerContext plannerContext)
+ {
+ switch (feature) {
+ case CAN_SELECT:
+ case ALLOW_BINDABLE_PLAN:
+ case TIMESERIES_QUERY:
+ case TOPN_QUERY:
+ return true;
+ case TIME_BOUNDARY_QUERY:
+ return QueryContexts.isTimeBoundaryPlanningEnabled(plannerContext.getQueryContext().getMergedParams());
+ case CAN_INSERT:
+ case CAN_REPLACE:
+ case READ_EXTERNAL_DATA:
+ case SCAN_ORDER_BY_NON_TIME:
+ case SCAN_NEEDS_SIGNATURE:
+ return false;
+ default:
+ throw new IAE("Unrecognized feature: %s", feature);
+ }
+ }
+
+ @Override
+ public QueryMaker buildQueryMakerForSelect(final RelRoot relRoot, final PlannerContext plannerContext)
+ {
+ return new NativeQueryMaker(
+ queryLifecycleFactory,
+ plannerContext,
+ jsonMapper,
+ relRoot.fields
+ );
+ }
+
+ @Override
+ public QueryMaker buildQueryMakerForInsert(
+ final String targetDataSource,
+ final RelRoot relRoot,
+ final PlannerContext plannerContext
+ )
+ {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/QueryFeatureInspector.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/QueryFeatureInspector.java
deleted file mode 100644
index a0d9a11241..0000000000
--- a/sql/src/main/java/org/apache/druid/sql/calcite/run/QueryFeatureInspector.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.sql.calcite.run;
-
-/**
- * Gives the SQL-to-Druid query translator information about what features are supporetd by the {@link QueryMaker}
- * that will execute the query.
- */
-public interface QueryFeatureInspector
-{
- /**
- * Returns whether a feature is present or not.
- */
- boolean feature(QueryFeature feature);
-}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/QueryMaker.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/QueryMaker.java
index c76504b2bd..72dae85d0e 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/run/QueryMaker.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/run/QueryMaker.java
@@ -19,24 +19,19 @@
package org.apache.druid.sql.calcite.run;
-import org.apache.calcite.rel.type.RelDataType;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.sql.calcite.rel.DruidQuery;
/**
- * Interface for executing Druid queries. Each one is created by a {@link QueryMakerFactory} and is tied to a
- * specific SQL query. Extends {@link QueryFeatureInspector}, so calling code can tell what this executor supports.
+ * Interface for executing Druid queries. Each one is created by a {@link SqlEngine} and is tied to a
+ * specific SQL query.
*/
-public interface QueryMaker extends QueryFeatureInspector
+public interface QueryMaker
{
- /**
- * Returns the SQL row type for this query.
- */
- RelDataType getResultType();
-
/**
* Executes a given Druid query, which is expected to correspond to the SQL query that this QueryMaker was originally
- * created for. The returned arrays match the row type given by {@link #getResultType()}.
+ * created for. The returned arrays match the row type given by {@link SqlEngine#resultTypeForSelect} or
+ * {@link SqlEngine#resultTypeForInsert}, depending on the nature of the statement.
*/
Sequence<Object[]> runQuery(DruidQuery druidQuery);
}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/QueryMakerFactory.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/QueryMakerFactory.java
deleted file mode 100644
index 0c7ffcdfba..0000000000
--- a/sql/src/main/java/org/apache/druid/sql/calcite/run/QueryMakerFactory.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.sql.calcite.run;
-
-import org.apache.calcite.rel.RelRoot;
-import org.apache.calcite.tools.ValidationException;
-import org.apache.druid.sql.calcite.planner.PlannerContext;
-
-/**
- * Interface for creating {@link QueryMaker}, which in turn are used to execute Druid queries.
- */
-public interface QueryMakerFactory
-{
- /**
- * Create a {@link QueryMaker} for a SELECT query.
- *
- * @param relRoot planned and validated rel
- * @param plannerContext context for this query
- *
- * @return an executor for the provided query
- *
- * @throws ValidationException if this factory cannot build an executor for the provided query
- */
- @SuppressWarnings("RedundantThrows")
- QueryMaker buildForSelect(RelRoot relRoot, PlannerContext plannerContext) throws ValidationException;
-
- /**
- * Create a {@link QueryMaker} for an INSERT ... SELECT query.
- *
- * @param targetDataSource datasource for the INSERT portion of the query
- * @param relRoot planned and validated rel for the SELECT portion of the query
- * @param plannerContext context for this query
- *
- * @return an executor for the provided query
- *
- * @throws ValidationException if this factory cannot build an executor for the provided query
- */
- QueryMaker buildForInsert(
- String targetDataSource,
- RelRoot relRoot,
- PlannerContext plannerContext
- ) throws ValidationException;
-}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/SqlEngine.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/SqlEngine.java
new file mode 100644
index 0000000000..896cdd8db3
--- /dev/null
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/run/SqlEngine.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.calcite.run;
+
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.tools.ValidationException;
+import org.apache.druid.query.QueryContext;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
+
+/**
+ * Engine for running SQL queries.
+ */
+public interface SqlEngine
+{
+ /**
+ * Name of this engine. Appears in user-visible messages.
+ */
+ String name();
+
+ /**
+ * Whether a feature applies to this engine or not.
+ */
+ boolean feature(EngineFeature feature, PlannerContext plannerContext);
+
+ /**
+ * Validates a provided query context. Returns quietly if the context is OK; throws {@link ValidationException}
+ * if the context has a problem.
+ */
+ void validateContext(QueryContext queryContext) throws ValidationException;
+
+ /**
+ * SQL row type that would be emitted by the {@link QueryMaker} from {@link #buildQueryMakerForSelect}.
+ * Called for SELECT. Not called for EXPLAIN, which is handled by the planner itself.
+ *
+ * @param typeFactory type factory
+ * @param validatedRowType row type from Calcite's validator
+ */
+ RelDataType resultTypeForSelect(RelDataTypeFactory typeFactory, RelDataType validatedRowType);
+
+ /**
+ * SQL row type that would be emitted by the {@link QueryMaker} from {@link #buildQueryMakerForInsert}.
+ * Called for INSERT and REPLACE. Not called for EXPLAIN, which is handled by the planner itself.
+ *
+ * @param typeFactory type factory
+ * @param validatedRowType row type from Calcite's validator
+ */
+ RelDataType resultTypeForInsert(RelDataTypeFactory typeFactory, RelDataType validatedRowType);
+
+ /**
+ * Create a {@link QueryMaker} for a SELECT query.
+ *
+ * @param relRoot planned and validated rel
+ * @param plannerContext context for this query
+ *
+ * @return an executor for the provided query
+ *
+ * @throws ValidationException if this engine cannot build an executor for the provided query
+ */
+ @SuppressWarnings("RedundantThrows")
+ QueryMaker buildQueryMakerForSelect(RelRoot relRoot, PlannerContext plannerContext) throws ValidationException;
+
+ /**
+ * Create a {@link QueryMaker} for an INSERT ... SELECT query.
+ *
+ * @param targetDataSource datasource for the INSERT portion of the query
+ * @param relRoot planned and validated rel for the SELECT portion of the query
+ * @param plannerContext context for this query
+ *
+ * @return an executor for the provided query
+ *
+ * @throws ValidationException if this engine cannot build an executor for the provided query
+ */
+ @SuppressWarnings("RedundantThrows")
+ QueryMaker buildQueryMakerForInsert(
+ String targetDataSource,
+ RelRoot relRoot,
+ PlannerContext plannerContext
+ ) throws ValidationException;
+}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/SqlEngines.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/SqlEngines.java
new file mode 100644
index 0000000000..f01a4714f4
--- /dev/null
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/run/SqlEngines.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.calcite.run;
+
+import org.apache.calcite.tools.ValidationException;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.query.QueryContext;
+
+import java.util.Set;
+
+public class SqlEngines
+{
+ /**
+ * Validates that a provided query context does not have any of the internal, special context keys listed in the
+ * {@code specialContextKeys} collection.
+ *
+ * Returns quietly if the context is OK; throws {@link ValidationException} if there is a problem.
+ *
+ * This is a helper function used by {@link SqlEngine#validateContext} implementations.
+ */
+ public static void validateNoSpecialContextKeys(final QueryContext queryContext, final Set<String> specialContextKeys)
+ throws ValidationException
+ {
+ for (String contextParameterName : queryContext.getMergedParams().keySet()) {
+ if (specialContextKeys.contains(contextParameterName)) {
+ throw new ValidationException(
+ StringUtils.format(
+ "Cannot execute query with context parameter [%s]",
+ contextParameterName
+ )
+ );
+ }
+ }
+ }
+}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/view/DruidViewMacro.java b/sql/src/main/java/org/apache/druid/sql/calcite/view/DruidViewMacro.java
index b4739cb9a4..7703427d37 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/view/DruidViewMacro.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/view/DruidViewMacro.java
@@ -57,9 +57,10 @@ public class DruidViewMacro implements TableMacro
public TranslatableTable apply(final List<Object> arguments)
{
final RelDataType rowType;
- try (final DruidPlanner planner = plannerFactory.createPlanner(viewSql, new QueryContext())) {
+ try (final DruidPlanner planner =
+ plannerFactory.createPlanner(ViewSqlEngine.INSTANCE, viewSql, new QueryContext())) {
planner.validate();
- rowType = planner.prepare().getRowType();
+ rowType = planner.prepare().getValidatedRowType();
}
catch (Exception e) {
throw new RuntimeException(e);
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/view/ViewSqlEngine.java b/sql/src/main/java/org/apache/druid/sql/calcite/view/ViewSqlEngine.java
new file mode 100644
index 0000000000..06395c7f5e
--- /dev/null
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/view/ViewSqlEngine.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.calcite.view;
+
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.query.QueryContext;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.sql.calcite.run.EngineFeature;
+import org.apache.druid.sql.calcite.run.QueryMaker;
+import org.apache.druid.sql.calcite.run.SqlEngine;
+
+/**
+ * Engine used for getting the row type of views. Does not do any actual planning or execution of the view.
+ */
+public class ViewSqlEngine implements SqlEngine
+{
+ public static final ViewSqlEngine INSTANCE = new ViewSqlEngine();
+ private static final String NAME = "view";
+
+ private ViewSqlEngine()
+ {
+ // Singleton.
+ }
+
+ @Override
+ public String name()
+ {
+ return NAME;
+ }
+
+ @Override
+ public boolean feature(EngineFeature feature, PlannerContext plannerContext)
+ {
+ switch (feature) {
+ // Use most permissive set of SELECT features, since our goal is to get the row type of the view.
+ // Later on, the query involving the view will be executed with an actual engine with a different set of
+ // features, and planning may fail then. But we don't want it to fail now.
+ case CAN_SELECT:
+ case ALLOW_BINDABLE_PLAN:
+ case READ_EXTERNAL_DATA:
+ case SCAN_ORDER_BY_NON_TIME:
+ return true;
+
+ // Views can't sit on top of INSERT or REPLACE.
+ case CAN_INSERT:
+ case CAN_REPLACE:
+ return false;
+
+ // Simplify planning by sticking to basic query types.
+ case TOPN_QUERY:
+ case TIMESERIES_QUERY:
+ case TIME_BOUNDARY_QUERY:
+ case SCAN_NEEDS_SIGNATURE:
+ return false;
+
+ default:
+ throw new IAE("Unrecognized feature: %s", feature);
+ }
+ }
+
+ @Override
+ public void validateContext(QueryContext queryContext)
+ {
+ // No query context validation for view row typing.
+ }
+
+ @Override
+ public RelDataType resultTypeForSelect(RelDataTypeFactory typeFactory, RelDataType validatedRowType)
+ {
+ return validatedRowType;
+ }
+
+ @Override
+ public RelDataType resultTypeForInsert(RelDataTypeFactory typeFactory, RelDataType validatedRowType)
+ {
+ // Can't have views of INSERT or REPLACE statements.
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public QueryMaker buildQueryMakerForSelect(RelRoot relRoot, PlannerContext plannerContext)
+ {
+ // View engine does not execute queries.
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public QueryMaker buildQueryMakerForInsert(String targetDataSource, RelRoot relRoot, PlannerContext plannerContext)
+ {
+ // Can't have views of INSERT or REPLACE statements.
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/sql/src/main/java/org/apache/druid/sql/http/SqlResource.java b/sql/src/main/java/org/apache/druid/sql/http/SqlResource.java
index 7c7b1c7c23..6a3787b7ea 100644
--- a/sql/src/main/java/org/apache/druid/sql/http/SqlResource.java
+++ b/sql/src/main/java/org/apache/druid/sql/http/SqlResource.java
@@ -21,6 +21,7 @@ package org.apache.druid.sql.http;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.io.CountingOutputStream;
import com.google.inject.Inject;
@@ -50,6 +51,8 @@ import org.apache.druid.sql.SqlLifecycleManager.Cancelable;
import org.apache.druid.sql.SqlPlanningException;
import org.apache.druid.sql.SqlRowTransformer;
import org.apache.druid.sql.SqlStatementFactory;
+import org.apache.druid.sql.SqlStatementFactoryFactory;
+import org.apache.druid.sql.calcite.run.NativeSqlEngine;
import javax.annotation.Nullable;
import javax.servlet.http.HttpServletRequest;
@@ -64,7 +67,6 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import javax.ws.rs.core.StreamingOutput;
-
import java.io.IOException;
import java.util.List;
import java.util.Set;
@@ -80,7 +82,7 @@ public class SqlResource
private final ObjectMapper jsonMapper;
private final AuthorizerMapper authorizerMapper;
- private final SqlStatementFactory sqlLifecycleFactory;
+ private final SqlStatementFactory sqlStatementFactory;
private final SqlLifecycleManager sqlLifecycleManager;
private final ServerConfig serverConfig;
@@ -88,16 +90,35 @@ public class SqlResource
public SqlResource(
@Json ObjectMapper jsonMapper,
AuthorizerMapper authorizerMapper,
- SqlStatementFactory sqlLifecycleFactory,
+ NativeSqlEngine engine,
+ SqlStatementFactoryFactory sqlStatementFactoryFactory,
SqlLifecycleManager sqlLifecycleManager,
ServerConfig serverConfig
)
+ {
+ this(
+ jsonMapper,
+ authorizerMapper,
+ sqlStatementFactoryFactory.factorize(engine),
+ sqlLifecycleManager,
+ serverConfig
+ );
+ }
+
+ @VisibleForTesting
+ SqlResource(
+ final ObjectMapper jsonMapper,
+ final AuthorizerMapper authorizerMapper,
+ final SqlStatementFactory sqlStatementFactory,
+ final SqlLifecycleManager sqlLifecycleManager,
+ final ServerConfig serverConfig
+ )
{
this.jsonMapper = Preconditions.checkNotNull(jsonMapper, "jsonMapper");
this.authorizerMapper = Preconditions.checkNotNull(authorizerMapper, "authorizerMapper");
- this.sqlLifecycleFactory = Preconditions.checkNotNull(sqlLifecycleFactory, "sqlLifecycleFactory");
+ this.sqlStatementFactory = Preconditions.checkNotNull(sqlStatementFactory, "sqlStatementFactory");
this.sqlLifecycleManager = Preconditions.checkNotNull(sqlLifecycleManager, "sqlLifecycleManager");
- this.serverConfig = serverConfig;
+ this.serverConfig = Preconditions.checkNotNull(serverConfig, "serverConfig");
}
@POST
@@ -108,7 +129,7 @@ public class SqlResource
@Context final HttpServletRequest req
) throws IOException
{
- final HttpStatement stmt = sqlLifecycleFactory.httpStatement(sqlQuery, req);
+ final HttpStatement stmt = sqlStatementFactory.httpStatement(sqlQuery, req);
final String sqlQueryId = stmt.sqlQueryId();
final String currThreadName = Thread.currentThread().getName();
diff --git a/sql/src/main/resources/saffron.properties b/sql/src/main/resources/saffron.properties
index ec5213c7da..505061d695 100644
--- a/sql/src/main/resources/saffron.properties
+++ b/sql/src/main/resources/saffron.properties
@@ -19,10 +19,9 @@
# Properties for Calcite (formerly known as "Saffron").
# Set here to ensure that the properties are absolutely,
# positively read when Calcite first initializes.
-#
-# This file _should_ be redundant with Calcites, but
-# there do seem to be race conditions at various times.
calcite.default.charset=UTF-16LE
calcite.default.nationalcharset=UTF-16LE
calcite.default.collation.name=UTF-16LE$en_US
+calcite.volcano.dump.sets=false
+calcite.volcano.dump.graphviz=false
diff --git a/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java b/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java
index dff9c52e33..41f9129032 100644
--- a/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java
@@ -68,7 +68,6 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import javax.servlet.http.HttpServletRequest;
-
import java.io.IOException;
import java.util.Collections;
import java.util.List;
@@ -89,7 +88,7 @@ public class SqlStatementTest
private SpecificSegmentsQuerySegmentWalker walker = null;
private TestRequestLogger testRequestLogger;
private ListeningExecutorService executorService;
- private SqlStatementFactory sqlLifecycleFactory;
+ private SqlStatementFactory sqlStatementFactory;
private final DefaultQueryConfig defaultQueryConfig = new DefaultQueryConfig(
ImmutableMap.of("DEFAULT_KEY", "DEFAULT_VALUE"));
@@ -145,7 +144,6 @@ public class SqlStatementTest
final PlannerFactory plannerFactory = new PlannerFactory(
rootSchema,
- CalciteTests.createMockQueryMakerFactory(walker, conglomerate),
operatorTable,
macroTable,
plannerConfig,
@@ -155,7 +153,7 @@ public class SqlStatementTest
new CalciteRulesManager(ImmutableSet.of())
);
- this.sqlLifecycleFactory = new SqlStatementFactory(
+ this.sqlStatementFactory = new SqlStatementFactoryFactory(
plannerFactory,
new NoopServiceEmitter(),
testRequestLogger,
@@ -163,7 +161,7 @@ public class SqlStatementTest
new AuthConfig(),
Suppliers.ofInstance(defaultQueryConfig),
new SqlLifecycleManager()
- );
+ ).factorize(CalciteTests.createMockSqlEngine(walker, conglomerate));
}
@After
@@ -214,7 +212,7 @@ public class SqlStatementTest
SqlQueryPlus sqlReq = queryPlus(
"SELECT COUNT(*) AS cnt, 'foo' AS TheFoo FROM druid.foo",
CalciteTests.REGULAR_USER_AUTH_RESULT);
- DirectStatement stmt = sqlLifecycleFactory.directStatement(sqlReq);
+ DirectStatement stmt = sqlStatementFactory.directStatement(sqlReq);
List<Object[]> results = stmt.execute().toList();
assertEquals(1, results.size());
assertEquals(6L, results.get(0)[0]);
@@ -227,7 +225,7 @@ public class SqlStatementTest
SqlQueryPlus sqlReq = queryPlus(
"SELECT COUNT(*) AS cnt, 'foo' AS",
CalciteTests.REGULAR_USER_AUTH_RESULT);
- DirectStatement stmt = sqlLifecycleFactory.directStatement(sqlReq);
+ DirectStatement stmt = sqlStatementFactory.directStatement(sqlReq);
try {
stmt.execute();
fail();
@@ -244,7 +242,7 @@ public class SqlStatementTest
SqlQueryPlus sqlReq = queryPlus(
"SELECT COUNT(*) AS cnt, 'foo' AS TheFoo FROM druid.bogus",
CalciteTests.REGULAR_USER_AUTH_RESULT);
- DirectStatement stmt = sqlLifecycleFactory.directStatement(sqlReq);
+ DirectStatement stmt = sqlStatementFactory.directStatement(sqlReq);
try {
stmt.execute();
fail();
@@ -261,7 +259,7 @@ public class SqlStatementTest
SqlQueryPlus sqlReq = queryPlus(
"select count(*) from forbiddenDatasource",
CalciteTests.REGULAR_USER_AUTH_RESULT);
- DirectStatement stmt = sqlLifecycleFactory.directStatement(sqlReq);
+ DirectStatement stmt = sqlStatementFactory.directStatement(sqlReq);
try {
stmt.execute();
fail();
@@ -290,7 +288,7 @@ public class SqlStatementTest
@Test
public void testHttpHappyPath()
{
- HttpStatement stmt = sqlLifecycleFactory.httpStatement(
+ HttpStatement stmt = sqlStatementFactory.httpStatement(
makeQuery("SELECT COUNT(*) AS cnt, 'foo' AS TheFoo FROM druid.foo"),
request(true)
);
@@ -303,7 +301,7 @@ public class SqlStatementTest
@Test
public void testHttpSyntaxError()
{
- HttpStatement stmt = sqlLifecycleFactory.httpStatement(
+ HttpStatement stmt = sqlStatementFactory.httpStatement(
makeQuery("SELECT COUNT(*) AS cnt, 'foo' AS"),
request(true)
);
@@ -320,7 +318,7 @@ public class SqlStatementTest
@Test
public void testHttpValidationError()
{
- HttpStatement stmt = sqlLifecycleFactory.httpStatement(
+ HttpStatement stmt = sqlStatementFactory.httpStatement(
makeQuery("SELECT COUNT(*) AS cnt, 'foo' AS TheFoo FROM druid.bogus"),
request(true)
);
@@ -337,7 +335,7 @@ public class SqlStatementTest
@Test
public void testHttpPermissionError()
{
- HttpStatement stmt = sqlLifecycleFactory.httpStatement(
+ HttpStatement stmt = sqlStatementFactory.httpStatement(
makeQuery("select count(*) from forbiddenDatasource"),
request(false)
);
@@ -359,10 +357,10 @@ public class SqlStatementTest
SqlQueryPlus sqlReq = queryPlus(
"SELECT COUNT(*) AS cnt, 'foo' AS TheFoo FROM druid.foo",
CalciteTests.REGULAR_USER_AUTH_RESULT);
- PreparedStatement stmt = sqlLifecycleFactory.preparedStatement(sqlReq);
+ PreparedStatement stmt = sqlStatementFactory.preparedStatement(sqlReq);
PrepareResult prepareResult = stmt.prepare();
- RelDataType rowType = prepareResult.getRowType();
+ RelDataType rowType = prepareResult.getReturnedRowType();
assertEquals(2, rowType.getFieldCount());
List<RelDataTypeField> fields = rowType.getFieldList();
assertEquals("cnt", fields.get(0).getName());
@@ -388,7 +386,7 @@ public class SqlStatementTest
SqlQueryPlus sqlReq = queryPlus(
"SELECT COUNT(*) AS cnt, 'foo' AS",
CalciteTests.REGULAR_USER_AUTH_RESULT);
- PreparedStatement stmt = sqlLifecycleFactory.preparedStatement(sqlReq);
+ PreparedStatement stmt = sqlStatementFactory.preparedStatement(sqlReq);
try {
stmt.prepare();
fail();
@@ -405,7 +403,7 @@ public class SqlStatementTest
SqlQueryPlus sqlReq = queryPlus(
"SELECT COUNT(*) AS cnt, 'foo' AS TheFoo FROM druid.bogus",
CalciteTests.REGULAR_USER_AUTH_RESULT);
- PreparedStatement stmt = sqlLifecycleFactory.preparedStatement(sqlReq);
+ PreparedStatement stmt = sqlStatementFactory.preparedStatement(sqlReq);
try {
stmt.prepare();
fail();
@@ -422,7 +420,7 @@ public class SqlStatementTest
SqlQueryPlus sqlReq = queryPlus(
"select count(*) from forbiddenDatasource",
CalciteTests.REGULAR_USER_AUTH_RESULT);
- PreparedStatement stmt = sqlLifecycleFactory.preparedStatement(sqlReq);
+ PreparedStatement stmt = sqlStatementFactory.preparedStatement(sqlReq);
try {
stmt.prepare();
fail();
@@ -443,7 +441,7 @@ public class SqlStatementTest
.context(ImmutableMap.of(QueryContexts.BY_SEGMENT_KEY, "true"))
.auth(CalciteTests.REGULAR_USER_AUTH_RESULT)
.build();
- DirectStatement stmt = sqlLifecycleFactory.directStatement(sqlReq);
+ DirectStatement stmt = sqlStatementFactory.directStatement(sqlReq);
Map<String, Object> context = stmt.sqlRequest().context().getMergedParams();
Assert.assertEquals(2, context.size());
// should contain only query id, not bySegment since it is not valid for SQL
@@ -458,7 +456,7 @@ public class SqlStatementTest
.context(ImmutableMap.of(QueryContexts.BY_SEGMENT_KEY, "true"))
.auth(CalciteTests.REGULAR_USER_AUTH_RESULT)
.build();
- DirectStatement stmt = sqlLifecycleFactory.directStatement(sqlReq);
+ DirectStatement stmt = sqlStatementFactory.directStatement(sqlReq);
Map<String, Object> context = stmt.sqlRequest().context().getMergedParams();
Assert.assertEquals(2, context.size());
// Statement should contain default query context values
diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java
index 858a277371..bc439b660f 100644
--- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java
@@ -46,6 +46,7 @@ import org.apache.druid.initialization.CoreInjectorBuilder;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.concurrent.Execs;
import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.math.expr.ExprMacroTable;
@@ -71,8 +72,8 @@ import org.apache.druid.sql.calcite.planner.Calcites;
import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.planner.PlannerFactory;
-import org.apache.druid.sql.calcite.run.NativeQueryMakerFactory;
-import org.apache.druid.sql.calcite.run.QueryMakerFactory;
+import org.apache.druid.sql.calcite.run.NativeSqlEngine;
+import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.schema.DruidSchemaName;
import org.apache.druid.sql.calcite.schema.NamedSchema;
@@ -116,6 +117,7 @@ import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadLocalRandom;
/**
@@ -219,7 +221,7 @@ public class DruidAvaticaHandlerTest extends CalciteTestBase
binder.bind(QueryScheduler.class)
.toProvider(QuerySchedulerProvider.class)
.in(LazySingleton.class);
- binder.bind(QueryMakerFactory.class).to(NativeQueryMakerFactory.class);
+ binder.bind(SqlEngine.class).to(NativeSqlEngine.class);
binder.bind(new TypeLiteral<Supplier<DefaultQueryConfig>>(){}).toInstance(Suppliers.ofInstance(new DefaultQueryConfig(ImmutableMap.of())));
binder.bind(CalciteRulesManager.class).toInstance(new CalciteRulesManager(ImmutableSet.of()));
}
@@ -916,25 +918,27 @@ public class DruidAvaticaHandlerTest extends CalciteTestBase
final DruidOperatorTable operatorTable = CalciteTests.createOperatorTable();
final ExprMacroTable macroTable = CalciteTests.createExprMacroTable();
final List<Meta.Frame> frames = new ArrayList<>();
+ final ScheduledExecutorService exec = Execs.scheduledSingleThreaded("testMaxRowsPerFrame");
DruidSchemaCatalog rootSchema =
CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
DruidMeta smallFrameDruidMeta = new DruidMeta(
- CalciteTests.createSqlLifecycleFactory(
- new PlannerFactory(
- rootSchema,
- CalciteTests.createMockQueryMakerFactory(walker, conglomerate),
- operatorTable,
- macroTable,
- plannerConfig,
- AuthTestUtils.TEST_AUTHORIZER_MAPPER,
- CalciteTests.getJsonMapper(),
- CalciteTests.DRUID_SCHEMA_NAME,
- new CalciteRulesManager(ImmutableSet.of())
- )
+ CalciteTests.createSqlStatementFactory(
+ CalciteTests.createMockSqlEngine(walker, conglomerate),
+ new PlannerFactory(
+ rootSchema,
+ operatorTable,
+ macroTable,
+ plannerConfig,
+ AuthTestUtils.TEST_AUTHORIZER_MAPPER,
+ CalciteTests.getJsonMapper(),
+ CalciteTests.DRUID_SCHEMA_NAME,
+ new CalciteRulesManager(ImmutableSet.of())
+ )
),
smallFrameConfig,
new ErrorHandler(new ServerConfig()),
- injector
+ exec,
+ injector.getInstance(AuthenticatorMapper.class).getAuthenticatorChain()
)
{
@Override
@@ -975,6 +979,8 @@ public class DruidAvaticaHandlerTest extends CalciteTestBase
),
rows
);
+
+ exec.shutdown();
}
@Test
@@ -1006,13 +1012,14 @@ public class DruidAvaticaHandlerTest extends CalciteTestBase
final DruidOperatorTable operatorTable = CalciteTests.createOperatorTable();
final ExprMacroTable macroTable = CalciteTests.createExprMacroTable();
final List<Meta.Frame> frames = new ArrayList<>();
+ final ScheduledExecutorService exec = Execs.scheduledSingleThreaded("testMaxRowsPerFrame");
DruidSchemaCatalog rootSchema =
CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
DruidMeta smallFrameDruidMeta = new DruidMeta(
- CalciteTests.createSqlLifecycleFactory(
+ CalciteTests.createSqlStatementFactory(
+ CalciteTests.createMockSqlEngine(walker, conglomerate),
new PlannerFactory(
rootSchema,
- CalciteTests.createMockQueryMakerFactory(walker, conglomerate),
operatorTable,
macroTable,
plannerConfig,
@@ -1024,7 +1031,8 @@ public class DruidAvaticaHandlerTest extends CalciteTestBase
),
smallFrameConfig,
new ErrorHandler(new ServerConfig()),
- injector
+ exec,
+ injector.getInstance(AuthenticatorMapper.class).getAuthenticatorChain()
)
{
@Override
@@ -1069,6 +1077,8 @@ public class DruidAvaticaHandlerTest extends CalciteTestBase
),
rows
);
+
+ exec.shutdown();
}
@Test
diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java
index 0528e6cf9f..2483dd663e 100644
--- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java
@@ -92,7 +92,7 @@ public class DruidStatementTest extends CalciteTestBase
}
private SpecificSegmentsQuerySegmentWalker walker;
- private SqlStatementFactory sqlLifecycleFactory;
+ private SqlStatementFactory sqlStatementFactory;
@Before
public void setUp() throws Exception
@@ -105,7 +105,6 @@ public class DruidStatementTest extends CalciteTestBase
CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
final PlannerFactory plannerFactory = new PlannerFactory(
rootSchema,
- CalciteTests.createMockQueryMakerFactory(walker, conglomerate),
operatorTable,
macroTable,
plannerConfig,
@@ -114,7 +113,10 @@ public class DruidStatementTest extends CalciteTestBase
CalciteTests.DRUID_SCHEMA_NAME,
new CalciteRulesManager(ImmutableSet.of())
);
- this.sqlLifecycleFactory = CalciteTests.createSqlLifecycleFactory(plannerFactory);
+ this.sqlStatementFactory = CalciteTests.createSqlStatementFactory(
+ CalciteTests.createMockSqlEngine(walker, conglomerate),
+ plannerFactory
+ );
}
@After
@@ -138,7 +140,7 @@ public class DruidStatementTest extends CalciteTestBase
"",
0,
new QueryContext(),
- sqlLifecycleFactory
+ sqlStatementFactory
);
}
@@ -516,7 +518,7 @@ public class DruidStatementTest extends CalciteTestBase
return new DruidJdbcPreparedStatement(
"",
0,
- sqlLifecycleFactory.preparedStatement(queryPlus),
+ sqlStatementFactory.preparedStatement(queryPlus),
Long.MAX_VALUE
);
}
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
index 1049228f7b..cbe8660182 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
@@ -96,6 +96,8 @@ import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.planner.PlannerContext;
import org.apache.druid.sql.calcite.planner.PlannerFactory;
+import org.apache.druid.sql.calcite.run.NativeSqlEngine;
+import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.schema.NoopDruidSchemaManager;
import org.apache.druid.sql.calcite.table.RowSignatures;
@@ -120,10 +122,10 @@ import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import javax.annotation.Nullable;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -259,6 +261,9 @@ public class BaseCalciteQueryTest extends CalciteTestBase
public static Closer resourceCloser;
public static int minTopNThreshold = TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD;
+ public final ObjectMapper queryJsonMapper;
+ @Nullable
+ public final SqlEngine engine0;
final boolean useDefault = NullHandling.replaceWithDefault();
@Rule
@@ -271,10 +276,21 @@ public class BaseCalciteQueryTest extends CalciteTestBase
public boolean cannotVectorize = false;
public boolean skipVectorize = false;
- public ObjectMapper queryJsonMapper;
public SpecificSegmentsQuerySegmentWalker walker = null;
+ public SqlEngine engine = null;
public QueryLogHook queryLogHook;
+ public BaseCalciteQueryTest()
+ {
+ this(null);
+ }
+
+ public BaseCalciteQueryTest(@Nullable final SqlEngine engine)
+ {
+ this.queryJsonMapper = createQueryJsonMapper();
+ this.engine0 = engine;
+ }
+
static {
TIMESERIES_CONTEXT_LOS_ANGELES.put(PlannerContext.CTX_SQL_QUERY_ID, DUMMY_SQL_ID);
TIMESERIES_CONTEXT_LOS_ANGELES.put(PlannerContext.CTX_SQL_CURRENT_TIMESTAMP, "2000-01-01T00:00:00Z");
@@ -470,7 +486,6 @@ public class BaseCalciteQueryTest extends CalciteTestBase
@Rule
public QueryLogHook getQueryLogHook()
{
- queryJsonMapper = createQueryJsonMapper();
return queryLogHook = QueryLogHook.create(queryJsonMapper);
}
@@ -478,6 +493,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
public void setUp() throws Exception
{
walker = createQuerySegmentWalker();
+ engine = createEngine();
// also register the static injected mapper, though across multiple test runs
ObjectMapper mapper = CalciteTests.getJsonMapper();
@@ -490,6 +506,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
{
walker.close();
walker = null;
+ engine = null;
}
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker() throws IOException
@@ -500,6 +517,18 @@ public class BaseCalciteQueryTest extends CalciteTestBase
);
}
+ public SqlEngine createEngine() throws IOException
+ {
+ if (engine0 == null) {
+ return new NativeSqlEngine(
+ CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate),
+ queryJsonMapper
+ );
+ } else {
+ return engine0;
+ }
+ }
+
public ObjectMapper createQueryJsonMapper()
{
// ugly workaround, for some reason the static mapper from Calcite.INJECTOR seems to get messed up over
@@ -533,15 +562,15 @@ public class BaseCalciteQueryTest extends CalciteTestBase
CalciteTests.INJECTOR.getInstance(LookupExtractorFactoryContainerProvider.class);
mapper.setInjectableValues(new InjectableValues.Std(injectables)
.addValue(ExprMacroTable.class.getName(), createMacroTable())
- .addValue(ObjectMapper.class.getName(), mapper)
- .addValue(
- DataSegment.PruneSpecsHolder.class,
- DataSegment.PruneSpecsHolder.DEFAULT
- )
- .addValue(
- LookupExtractorFactoryContainerProvider.class.getName(),
- lookupProvider
- )
+ .addValue(ObjectMapper.class.getName(), mapper)
+ .addValue(
+ DataSegment.PruneSpecsHolder.class,
+ DataSegment.PruneSpecsHolder.DEFAULT
+ )
+ .addValue(
+ LookupExtractorFactoryContainerProvider.class.getName(),
+ lookupProvider
+ )
);
}
@@ -571,9 +600,11 @@ public class BaseCalciteQueryTest extends CalciteTestBase
log.error(e, "Expected CannotPlanException for query: %s", sql);
Assert.fail(sql);
}
- Assert.assertEquals(sql,
- StringUtils.format("Cannot build plan for query: %s. %s", sql, expectedError),
- e.getMessage());
+ Assert.assertEquals(
+ sql,
+ StringUtils.format("Cannot build plan for query. %s", expectedError),
+ e.getMessage()
+ );
}
/**
@@ -728,11 +759,16 @@ public class BaseCalciteQueryTest extends CalciteTestBase
final List<Object[]> expectedResults
)
{
- log.info("SQL: %s", sql);
- queryLogHook.clearRecordedQueries();
- final Pair<RowSignature, List<Object[]>> plannerResults =
- getResults(plannerConfig, queryContext, DEFAULT_PARAMETERS, sql, authenticationResult);
- verifyResults(sql, expectedQueries, expectedResults, plannerResults);
+ testQuery(
+ plannerConfig,
+ queryContext,
+ Collections.emptyList(),
+ sql,
+ authenticationResult,
+ expectedQueries,
+ expectedResults,
+ null
+ );
}
public void testQuery(
@@ -780,6 +816,9 @@ public class BaseCalciteQueryTest extends CalciteTestBase
);
}
+ /**
+ * All testQuery roads lead to this method.
+ */
public void testQuery(
final PlannerConfig plannerConfig,
final Map<String, Object> queryContext,
@@ -824,7 +863,8 @@ public class BaseCalciteQueryTest extends CalciteTestBase
expectedExceptionInitializer.accept(expectedException);
}
- final Pair<RowSignature, List<Object[]>> plannerResults = getResults(plannerConfig, theQueryContext, parameters, sql, authenticationResult);
+ final Pair<RowSignature, List<Object[]>> plannerResults =
+ getResults(plannerConfig, theQueryContext, parameters, sql, authenticationResult);
verifyResults(sql, theQueries, plannerResults, expectedResultsVerifier);
}
}
@@ -850,7 +890,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
);
}
- public Pair<RowSignature, List<Object[]>> getResults(
+ private Pair<RowSignature, List<Object[]>> getResults(
final PlannerConfig plannerConfig,
final Map<String, Object> queryContext,
final List<SqlParameter> parameters,
@@ -862,7 +902,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
final ObjectMapper objectMapper
)
{
- final SqlStatementFactory sqlLifecycleFactory = getSqlLifecycleFactory(
+ final SqlStatementFactory sqlStatementFactory = getSqlStatementFactory(
plannerConfig,
new AuthConfig(),
operatorTable,
@@ -870,7 +910,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
authorizerMapper,
objectMapper
);
- final DirectStatement stmt = sqlLifecycleFactory.directStatement(
+ final DirectStatement stmt = sqlStatementFactory.directStatement(
SqlQueryPlus.builder(sql)
.context(queryContext)
.sqlParameters(parameters)
@@ -878,7 +918,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
.build()
);
Sequence<Object[]> results = stmt.execute();
- RelDataType rowType = stmt.prepareResult().getRowType();
+ RelDataType rowType = stmt.prepareResult().getReturnedRowType();
return new Pair<>(
RowSignatures.fromRelDataType(rowType.getFieldNames(), rowType),
results.toList()
@@ -1001,7 +1041,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
AuthenticationResult authenticationResult
)
{
- SqlStatementFactory lifecycleFactory = getSqlLifecycleFactory(
+ SqlStatementFactory lifecycleFactory = getSqlStatementFactory(
plannerConfig,
authConfig,
createOperatorTable(),
@@ -1019,7 +1059,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
return stmt.allResources();
}
- public SqlStatementFactory getSqlLifecycleFactory(
+ public SqlStatementFactory getSqlStatementFactory(
PlannerConfig plannerConfig,
AuthConfig authConfig,
DruidOperatorTable operatorTable,
@@ -1040,10 +1080,6 @@ public class BaseCalciteQueryTest extends CalciteTestBase
final PlannerFactory plannerFactory = new PlannerFactory(
rootSchema,
- new TestQueryMakerFactory(
- CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate),
- objectMapper
- ),
operatorTable,
macroTable,
plannerConfig,
@@ -1052,7 +1088,11 @@ public class BaseCalciteQueryTest extends CalciteTestBase
CalciteTests.DRUID_SCHEMA_NAME,
new CalciteRulesManager(ImmutableSet.of())
);
- final SqlStatementFactory sqlLifecycleFactory = CalciteTests.createSqlLifecycleFactory(plannerFactory, authConfig);
+ final SqlStatementFactory sqlStatementFactory = CalciteTests.createSqlStatementFactory(
+ engine,
+ plannerFactory,
+ authConfig
+ );
viewManager.createView(
plannerFactory,
@@ -1097,7 +1137,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
"invalidView",
"SELECT __time, dim1, dim2, m1 FROM druid.invalidDatasource WHERE dim2 = 'a'"
);
- return sqlLifecycleFactory;
+ return sqlStatementFactory;
}
protected void cannotVectorize()
@@ -1214,7 +1254,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
}
/**
- * Reset the walker and conglomerate with required number of merge buffers. Default value is 2.
+ * Reset the conglomerate, walker, and engine with required number of merge buffers. Default value is 2.
*/
protected void requireMergeBuffers(int numMergeBuffers) throws IOException
{
@@ -1222,7 +1262,8 @@ public class BaseCalciteQueryTest extends CalciteTestBase
resourceCloser,
QueryStackTests.getProcessingConfig(true, numMergeBuffers)
);
- walker = CalciteTests.createMockWalker(conglomerate, temporaryFolder.newFolder());
+ walker = createQuerySegmentWalker();
+ engine = createEngine();
}
protected Map<String, Object> withTimestampResultContext(
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java
index a19dd3d038..daa63f3f49 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java
@@ -154,7 +154,7 @@ public class CalciteExplainQueryTest extends BaseCalciteQueryTest
// Skip vectorization since otherwise the "context" will change for each subtest.
skipVectorize();
String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTim [...]
- String legacyExplanationWithContext = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sq [...]
+ String legacyExplanationWithContext = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sq [...]
String explanation = "[{"
+ "\"query\":{\"queryType\":\"scan\","
+ "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},"
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java
index c862721dcb..72dd87dc9c 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java
@@ -92,6 +92,11 @@ public class CalciteIngestionDmlTest extends BaseCalciteQueryTest
protected boolean didTest = false;
+ public CalciteIngestionDmlTest()
+ {
+ super(IngestionTestSqlEngine.INSTANCE);
+ }
+
@After
@Override
public void tearDown() throws Exception
@@ -267,7 +272,7 @@ public class CalciteIngestionDmlTest extends BaseCalciteQueryTest
throw new ISE("Test must not have expectedQuery");
}
- final SqlStatementFactory sqlLifecycleFactory = getSqlLifecycleFactory(
+ final SqlStatementFactory sqlStatementFactory = getSqlStatementFactory(
plannerConfig,
new AuthConfig(),
createOperatorTable(),
@@ -276,7 +281,7 @@ public class CalciteIngestionDmlTest extends BaseCalciteQueryTest
queryJsonMapper
);
- DirectStatement stmt = sqlLifecycleFactory.directStatement(
+ DirectStatement stmt = sqlStatementFactory.directStatement(
SqlQueryPlus
.builder(sql)
.context(queryContext)
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java
index 2dc4ea3142..fccbf1682a 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java
@@ -28,6 +28,7 @@ import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.java.util.common.jackson.JacksonUtils;
+import org.apache.druid.query.QueryDataSource;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
@@ -35,6 +36,7 @@ import org.apache.druid.query.groupby.GroupByQuery;
import org.apache.druid.query.scan.ScanQuery;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.segment.join.JoinType;
import org.apache.druid.server.security.ForbiddenException;
import org.apache.druid.sql.SqlPlanningException;
import org.apache.druid.sql.calcite.external.ExternalDataSource;
@@ -51,6 +53,7 @@ import org.junit.Test;
import org.junit.internal.matchers.ThrowableMessageMatcher;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -80,7 +83,7 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
}
@Test
- public void testInsertFromView()
+ public void testInsertFromViewA()
{
testIngestionQuery()
.sql("INSERT INTO dst SELECT * FROM view.aview PARTITIONED BY ALL TIME")
@@ -99,6 +102,58 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
.verify();
}
+ @Test
+ public void testInsertFromViewC()
+ {
+ final RowSignature expectedSignature =
+ RowSignature.builder()
+ .add("dim1_firstchar", ColumnType.STRING)
+ .add("dim2", ColumnType.STRING)
+ .add("l2", ColumnType.LONG)
+ .build();
+
+ testIngestionQuery()
+ .sql("INSERT INTO dst SELECT * FROM view.cview PARTITIONED BY ALL TIME")
+ .expectTarget("dst", expectedSignature)
+ .expectResources(viewRead("cview"), dataSourceWrite("dst"))
+ .expectQuery(
+ newScanQueryBuilder()
+ .dataSource(
+ join(
+ new QueryDataSource(
+ newScanQueryBuilder()
+ .dataSource("foo")
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .filters(selector("dim2", "a", null))
+ .columns("dim1", "dim2")
+ .context(PARTITIONED_BY_ALL_TIME_QUERY_CONTEXT)
+ .build()
+ ),
+ new QueryDataSource(
+ newScanQueryBuilder()
+ .dataSource("numfoo")
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .columns("dim2", "l2")
+ .context(PARTITIONED_BY_ALL_TIME_QUERY_CONTEXT)
+ .build()
+ ),
+ "j0.",
+ "(\"dim2\" == \"j0.dim2\")",
+ JoinType.INNER
+ )
+ )
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .virtualColumns(
+ expressionVirtualColumn("v0", "substring(\"dim1\", 0, 1)", ColumnType.STRING),
+ expressionVirtualColumn("v1", "'a'", ColumnType.STRING)
+ )
+ .columns("j0.l2", "v0", "v1")
+ .context(PARTITIONED_BY_ALL_TIME_QUERY_CONTEXT)
+ .build()
+ )
+ .verify();
+ }
+
@Test
public void testInsertIntoExistingTable()
{
@@ -162,6 +217,20 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
.verify();
}
+ @Test
+ public void testSelectFromSystemTable()
+ {
+ // TestInsertSqlEngine does not include ALLOW_BINDABLE_PLAN, so cannot query system tables.
+
+ testIngestionQuery()
+ .sql("INSERT INTO dst SELECT * FROM INFORMATION_SCHEMA.COLUMNS PARTITIONED BY ALL TIME")
+ .expectValidationError(
+ SqlPlanningException.class,
+ "Cannot query table [INFORMATION_SCHEMA.COLUMNS] with SQL engine 'ingestion-test'."
+ )
+ .verify();
+ }
+
@Test
public void testInsertIntoSystemTable()
{
@@ -169,7 +238,7 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
.sql("INSERT INTO INFORMATION_SCHEMA.COLUMNS SELECT * FROM foo PARTITIONED BY ALL TIME")
.expectValidationError(
SqlPlanningException.class,
- "Cannot INSERT into [INFORMATION_SCHEMA.COLUMNS] because it is not a Druid datasource."
+ "Cannot INSERT into [INFORMATION_SCHEMA.COLUMNS] because it is not a Druid datasource (schema = druid)."
)
.verify();
}
@@ -181,7 +250,7 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
.sql("INSERT INTO view.aview SELECT * FROM foo PARTITIONED BY ALL TIME")
.expectValidationError(
SqlPlanningException.class,
- "Cannot INSERT into [view.aview] because it is not a Druid datasource."
+ "Cannot INSERT into [view.aview] because it is not a Druid datasource (schema = druid)."
)
.verify();
}
@@ -211,7 +280,7 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
.sql("INSERT INTO nonexistent.dst SELECT * FROM foo PARTITIONED BY ALL TIME")
.expectValidationError(
SqlPlanningException.class,
- "Cannot INSERT into [nonexistent.dst] because it is not a Druid datasource."
+ "Cannot INSERT into [nonexistent.dst] because it is not a Druid datasource (schema = druid)."
)
.verify();
}
@@ -364,7 +433,10 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
+ "SELECT __time, FLOOR(m1) as floor_m1, dim1, CEIL(m2) as ceil_m2 FROM foo "
+ "CLUSTERED BY 2, dim1 DESC, CEIL(m2)"
)
- .expectValidationError(SqlPlanningException.class, "CLUSTERED BY found before PARTITIONED BY. In druid, the CLUSTERED BY clause has to be specified after the PARTITIONED BY clause")
+ .expectValidationError(
+ SqlPlanningException.class,
+ "CLUSTERED BY found before PARTITIONED BY. In druid, the CLUSTERED BY clause has to be specified after the PARTITIONED BY clause"
+ )
.verify();
}
@@ -526,7 +598,7 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
.columns("x", "y", "z")
.context(
queryJsonMapper.readValue(
- "{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlInsertSegmentGranularity\":\"{\\\"type\\\":\\\"all\\\"}\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}",
+ "{\"sqlInsertSegmentGranularity\":\"{\\\"type\\\":\\\"all\\\"}\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}",
JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT
)
)
@@ -540,18 +612,24 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
// Use testQuery for EXPLAIN (not testIngestionQuery).
testQuery(
new PlannerConfig(),
+ ImmutableMap.of("sqlQueryId", "dummy"),
+ Collections.emptyList(),
StringUtils.format(
"EXPLAIN PLAN FOR INSERT INTO dst SELECT * FROM %s PARTITIONED BY ALL TIME",
externSql(externalDataSource)
),
CalciteTests.SUPER_USER_AUTH_RESULT,
ImmutableList.of(),
- ImmutableList.of(
- new Object[]{
- expectedExplanation,
- "[{\"name\":\"EXTERNAL\",\"type\":\"EXTERNAL\"},{\"name\":\"dst\",\"type\":\"DATASOURCE\"}]"
- }
- )
+ new DefaultResultsVerifier(
+ ImmutableList.of(
+ new Object[]{
+ expectedExplanation,
+ "[{\"name\":\"EXTERNAL\",\"type\":\"EXTERNAL\"},{\"name\":\"dst\",\"type\":\"DATASOURCE\"}]"
+ }
+ ),
+ null
+ ),
+ null
);
// Not using testIngestionQuery, so must set didTest manually to satisfy the check in tearDown.
@@ -579,6 +657,18 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
didTest = true;
}
+ @Test
+ public void testSurfaceErrorsWhenInsertingThroughIncorrectSelectStatment()
+ {
+ assertQueryIsUnplannable(
+ "INSERT INTO druid.dst SELECT dim2, dim1, m1 FROM foo2 UNION SELECT dim1, dim2, m1 FROM foo PARTITIONED BY ALL TIME",
+ "Possible error: SQL requires 'UNION' but only 'UNION ALL' is supported."
+ );
+
+ // Not using testIngestionQuery, so must set didTest manually to satisfy the check in tearDown.
+ didTest = true;
+ }
+
@Test
public void testInsertFromExternalUnauthorized()
{
@@ -788,7 +878,10 @@ public class CalciteInsertDmlTest extends CalciteIngestionDmlTest
testIngestionQuery()
.context(context)
.sql("INSERT INTO dst SELECT * FROM foo PARTITIONED BY ALL TIME")
- .expectValidationError(SqlPlanningException.class, "sqlOuterLimit cannot be provided on INSERT or REPLACE queries.")
+ .expectValidationError(
+ SqlPlanningException.class,
+ "sqlOuterLimit cannot be provided with INSERT."
+ )
.verify();
}
}
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index 8bdda17d27..15be7d30b2 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -118,6 +118,7 @@ import org.apache.druid.sql.calcite.planner.PlannerContext;
import org.apache.druid.sql.calcite.rel.CannotBuildQueryException;
import org.apache.druid.sql.calcite.util.CalciteTests;
import org.hamcrest.CoreMatchers;
+import org.hamcrest.MatcherAssert;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Interval;
@@ -355,6 +356,46 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
+ @Test
+ public void testCannotInsertWithNativeEngine()
+ {
+ final SqlPlanningException e = Assert.assertThrows(
+ SqlPlanningException.class,
+ () -> testQuery(
+ "INSERT INTO dst SELECT * FROM foo PARTITIONED BY ALL",
+ ImmutableList.of(),
+ ImmutableList.of()
+ )
+ );
+
+ MatcherAssert.assertThat(
+ e,
+ ThrowableMessageMatcher.hasMessage(
+ CoreMatchers.equalTo("Cannot execute INSERT with SQL engine 'native'.")
+ )
+ );
+ }
+
+ @Test
+ public void testCannotReplaceWithNativeEngine()
+ {
+ final SqlPlanningException e = Assert.assertThrows(
+ SqlPlanningException.class,
+ () -> testQuery(
+ "REPLACE INTO dst OVERWRITE ALL SELECT * FROM foo PARTITIONED BY ALL",
+ ImmutableList.of(),
+ ImmutableList.of()
+ )
+ );
+
+ MatcherAssert.assertThat(
+ e,
+ ThrowableMessageMatcher.hasMessage(
+ CoreMatchers.equalTo("Cannot execute REPLACE with SQL engine 'native'.")
+ )
+ );
+ }
+
@Test
public void testAggregatorsOnInformationSchemaColumns()
{
@@ -13890,15 +13931,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
- @Test
- public void testSurfaceErrorsWhenInsertingThroughIncorrectSelectStatment()
- {
- assertQueryIsUnplannable(
- "INSERT INTO druid.dst SELECT dim2, dim1, m1 FROM foo2 UNION SELECT dim1, dim2, m1 FROM foo PARTITIONED BY ALL TIME",
- "Possible error: SQL requires 'UNION' but only 'UNION ALL' is supported."
- );
- }
-
@Test
public void testPlanWithInFilterLessThanInSubQueryThreshold()
{
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java
index b0c1f37e2f..2cbd53b54a 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteReplaceDmlTest.java
@@ -45,6 +45,7 @@ import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -436,7 +437,7 @@ public class CalciteReplaceDmlTest extends CalciteIngestionDmlTest
.sql("REPLACE INTO INFORMATION_SCHEMA.COLUMNS OVERWRITE ALL SELECT * FROM foo PARTITIONED BY ALL TIME")
.expectValidationError(
SqlPlanningException.class,
- "Cannot REPLACE into [INFORMATION_SCHEMA.COLUMNS] because it is not a Druid datasource."
+ "Cannot REPLACE into [INFORMATION_SCHEMA.COLUMNS] because it is not a Druid datasource (schema = druid)."
)
.verify();
}
@@ -448,7 +449,7 @@ public class CalciteReplaceDmlTest extends CalciteIngestionDmlTest
.sql("REPLACE INTO view.aview OVERWRITE ALL SELECT * FROM foo PARTITIONED BY ALL TIME")
.expectValidationError(
SqlPlanningException.class,
- "Cannot REPLACE into [view.aview] because it is not a Druid datasource."
+ "Cannot REPLACE into [view.aview] because it is not a Druid datasource (schema = druid)."
)
.verify();
}
@@ -478,7 +479,7 @@ public class CalciteReplaceDmlTest extends CalciteIngestionDmlTest
.sql("REPLACE INTO nonexistent.dst OVERWRITE ALL SELECT * FROM foo PARTITIONED BY ALL TIME")
.expectValidationError(
SqlPlanningException.class,
- "Cannot REPLACE into [nonexistent.dst] because it is not a Druid datasource."
+ "Cannot REPLACE into [nonexistent.dst] because it is not a Druid datasource (schema = druid)."
)
.verify();
}
@@ -605,7 +606,7 @@ public class CalciteReplaceDmlTest extends CalciteIngestionDmlTest
.columns("x", "y", "z")
.context(
queryJsonMapper.readValue(
- "{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlInsertSegmentGranularity\":\"{\\\"type\\\":\\\"all\\\"}\",\"sqlQueryId\":\"dummy\",\"sqlReplaceTimeChunks\":\"all\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}",
+ "{\"sqlInsertSegmentGranularity\":\"{\\\"type\\\":\\\"all\\\"}\",\"sqlQueryId\":\"dummy\",\"sqlReplaceTimeChunks\":\"all\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}",
JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT
)
)
@@ -619,18 +620,24 @@ public class CalciteReplaceDmlTest extends CalciteIngestionDmlTest
// Use testQuery for EXPLAIN (not testIngestionQuery).
testQuery(
new PlannerConfig(),
+ ImmutableMap.of("sqlQueryId", "dummy"),
+ Collections.emptyList(),
StringUtils.format(
"EXPLAIN PLAN FOR REPLACE INTO dst OVERWRITE ALL SELECT * FROM %s PARTITIONED BY ALL TIME",
externSql(externalDataSource)
),
CalciteTests.SUPER_USER_AUTH_RESULT,
ImmutableList.of(),
- ImmutableList.of(
- new Object[]{
- expectedExplanation,
- "[{\"name\":\"EXTERNAL\",\"type\":\"EXTERNAL\"},{\"name\":\"dst\",\"type\":\"DATASOURCE\"}]"
- }
- )
+ new DefaultResultsVerifier(
+ ImmutableList.of(
+ new Object[]{
+ expectedExplanation,
+ "[{\"name\":\"EXTERNAL\",\"type\":\"EXTERNAL\"},{\"name\":\"dst\",\"type\":\"DATASOURCE\"}]"
+ }
+ ),
+ null
+ ),
+ null
);
// Not using testIngestionQuery, so must set didTest manually to satisfy the check in tearDown.
@@ -761,7 +768,7 @@ public class CalciteReplaceDmlTest extends CalciteIngestionDmlTest
testIngestionQuery()
.context(context)
.sql("REPLACE INTO dst OVERWRITE ALL SELECT * FROM foo PARTITIONED BY ALL TIME")
- .expectValidationError(SqlPlanningException.class, "sqlOuterLimit cannot be provided on INSERT or REPLACE queries.")
+ .expectValidationError(SqlPlanningException.class, "sqlOuterLimit cannot be provided with REPLACE.")
.verify();
}
}
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteScanSignatureTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteScanSignatureTest.java
new file mode 100644
index 0000000000..50c6888c75
--- /dev/null
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteScanSignatureTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.calcite;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.tools.ValidationException;
+import org.apache.druid.query.QueryContext;
+import org.apache.druid.query.scan.ScanQuery;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.sql.calcite.filtration.Filtration;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.sql.calcite.rel.DruidQuery;
+import org.apache.druid.sql.calcite.run.EngineFeature;
+import org.apache.druid.sql.calcite.run.QueryMaker;
+import org.apache.druid.sql.calcite.run.SqlEngine;
+import org.apache.druid.sql.calcite.util.CalciteTests;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class CalciteScanSignatureTest extends BaseCalciteQueryTest
+{
+ public CalciteScanSignatureTest()
+ {
+ super(null);
+ }
+
+ @Test
+ public void testScanSignature()
+ {
+ final Map<String, Object> context = new HashMap<>(QUERY_CONTEXT_DEFAULT);
+ context.put(DruidQuery.CTX_SCAN_SIGNATURE, "[{\"name\":\"v0\",\"type\":\"STRING\"}]");
+
+ testQuery(
+ "SELECT CONCAT(dim1, '-', dim1, '_', dim1) as dimX FROM foo",
+ ImmutableList.of(
+ newScanQueryBuilder()
+ .dataSource(CalciteTests.DATASOURCE1)
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .virtualColumns(expressionVirtualColumn(
+ "v0",
+ "concat(\"dim1\",'-',\"dim1\",'_',\"dim1\")",
+ ColumnType.STRING
+ ))
+ .columns("v0")
+ .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
+ .context(context)
+ .build()
+ ),
+ ImmutableList.of(
+ new Object[]{"-_"},
+ new Object[]{"10.1-10.1_10.1"},
+ new Object[]{"2-2_2"},
+ new Object[]{"1-1_1"},
+ new Object[]{"def-def_def"},
+ new Object[]{"abc-abc_abc"}
+ )
+ );
+ }
+
+ @Override
+ public SqlEngine createEngine() throws IOException
+ {
+ // Create an engine that says yes to EngineFeature.SCAN_NEEDS_SIGNATURE.
+ return new ScanSignatureTestSqlEngine(super.createEngine());
+ }
+
+ private static class ScanSignatureTestSqlEngine implements SqlEngine
+ {
+ private final SqlEngine parent;
+
+ public ScanSignatureTestSqlEngine(final SqlEngine parent)
+ {
+ this.parent = parent;
+ }
+
+ @Override
+ public String name()
+ {
+ return getClass().getName();
+ }
+
+ @Override
+ public boolean feature(EngineFeature feature, PlannerContext plannerContext)
+ {
+ return feature == EngineFeature.SCAN_NEEDS_SIGNATURE || parent.feature(feature, plannerContext);
+ }
+
+ @Override
+ public void validateContext(QueryContext queryContext)
+ {
+ // No validation.
+ }
+
+ @Override
+ public RelDataType resultTypeForSelect(RelDataTypeFactory typeFactory, RelDataType validatedRowType)
+ {
+ return validatedRowType;
+ }
+
+ @Override
+ public RelDataType resultTypeForInsert(RelDataTypeFactory typeFactory, RelDataType validatedRowType)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public QueryMaker buildQueryMakerForSelect(RelRoot relRoot, PlannerContext plannerContext)
+ throws ValidationException
+ {
+ return parent.buildQueryMakerForSelect(relRoot, plannerContext);
+ }
+
+ @Override
+ public QueryMaker buildQueryMakerForInsert(String targetDataSource, RelRoot relRoot, PlannerContext plannerContext)
+ {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/IngestionTestSqlEngine.java b/sql/src/test/java/org/apache/druid/sql/calcite/IngestionTestSqlEngine.java
new file mode 100644
index 0000000000..a8c6132ddb
--- /dev/null
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/IngestionTestSqlEngine.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.calcite;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.query.QueryContext;
+import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.sql.calcite.run.EngineFeature;
+import org.apache.druid.sql.calcite.run.QueryMaker;
+import org.apache.druid.sql.calcite.run.SqlEngine;
+import org.apache.druid.sql.calcite.table.RowSignatures;
+
+public class IngestionTestSqlEngine implements SqlEngine
+{
+ public static final IngestionTestSqlEngine INSTANCE = new IngestionTestSqlEngine();
+
+ private IngestionTestSqlEngine()
+ {
+ }
+
+ @Override
+ public String name()
+ {
+ return "ingestion-test";
+ }
+
+ @Override
+ public void validateContext(QueryContext queryContext)
+ {
+ // No validation.
+ }
+
+ @Override
+ public RelDataType resultTypeForSelect(RelDataTypeFactory typeFactory, RelDataType validatedRowType)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public RelDataType resultTypeForInsert(RelDataTypeFactory typeFactory, RelDataType validatedRowType)
+ {
+ // Matches the return structure of TestInsertQueryMaker.
+ return typeFactory.createStructType(
+ ImmutableList.of(
+ typeFactory.createSqlType(SqlTypeName.VARCHAR),
+ typeFactory.createSqlType(SqlTypeName.OTHER)
+ ),
+ ImmutableList.of("dataSource", "signature")
+ );
+ }
+
+ @Override
+ public boolean feature(final EngineFeature feature, final PlannerContext plannerContext)
+ {
+ switch (feature) {
+ case CAN_SELECT:
+ case ALLOW_BINDABLE_PLAN:
+ case TIMESERIES_QUERY:
+ case TOPN_QUERY:
+ case TIME_BOUNDARY_QUERY:
+ case SCAN_NEEDS_SIGNATURE:
+ return false;
+ case CAN_INSERT:
+ case CAN_REPLACE:
+ case READ_EXTERNAL_DATA:
+ case SCAN_ORDER_BY_NON_TIME:
+ return true;
+ default:
+ throw new IAE("Unrecognized feature: %s", feature);
+ }
+ }
+
+ @Override
+ public QueryMaker buildQueryMakerForSelect(RelRoot relRoot, PlannerContext plannerContext)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public QueryMaker buildQueryMakerForInsert(String targetDataSource, RelRoot relRoot, PlannerContext plannerContext)
+ {
+ final RowSignature signature = RowSignatures.fromRelDataType(
+ relRoot.validatedRowType.getFieldNames(),
+ relRoot.validatedRowType
+ );
+
+ return new TestInsertQueryMaker(targetDataSource, signature);
+ }
+}
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java
index 621cfd2ddf..3823bd4af7 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java
@@ -31,6 +31,7 @@ import org.apache.druid.java.util.common.guava.Yielders;
import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.math.expr.ExpressionProcessing;
+import org.apache.druid.query.QueryContext;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.segment.QueryableIndex;
@@ -44,6 +45,7 @@ import org.apache.druid.sql.calcite.planner.DruidPlanner;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.planner.PlannerFactory;
import org.apache.druid.sql.calcite.planner.PlannerResult;
+import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.util.CalciteTests;
import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker;
@@ -58,10 +60,8 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import javax.annotation.Nullable;
-
import java.io.IOException;
import java.util.List;
-import java.util.Map;
import java.util.stream.Collectors;
@RunWith(Parameterized.class)
@@ -104,6 +104,7 @@ public class SqlVectorizedExpressionSanityTest extends InitializedNullHandlingTe
private static Closer CLOSER;
private static QueryRunnerFactoryConglomerate CONGLOMERATE;
private static SpecificSegmentsQuerySegmentWalker WALKER;
+ private static SqlEngine ENGINE;
@Nullable
private static PlannerFactory PLANNER_FACTORY;
@@ -138,9 +139,9 @@ public class SqlVectorizedExpressionSanityTest extends InitializedNullHandlingTe
final PlannerConfig plannerConfig = new PlannerConfig();
final DruidSchemaCatalog rootSchema =
CalciteTests.createMockRootSchema(CONGLOMERATE, WALKER, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
+ ENGINE = CalciteTests.createMockSqlEngine(WALKER, CONGLOMERATE);
PLANNER_FACTORY = new PlannerFactory(
rootSchema,
- CalciteTests.createMockQueryMakerFactory(WALKER, CONGLOMERATE),
CalciteTests.createOperatorTable(),
CalciteTests.createExprMacroTable(),
plannerConfig,
@@ -180,18 +181,22 @@ public class SqlVectorizedExpressionSanityTest extends InitializedNullHandlingTe
public static void sanityTestVectorizedSqlQueries(PlannerFactory plannerFactory, String query)
throws ValidationException
{
- final Map<String, Object> vector = ImmutableMap.of(
- QueryContexts.VECTORIZE_KEY, "force",
- QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, "force"
+ final QueryContext vector = new QueryContext(
+ ImmutableMap.of(
+ QueryContexts.VECTORIZE_KEY, "force",
+ QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, "force"
+ )
);
- final Map<String, Object> nonvector = ImmutableMap.of(
- QueryContexts.VECTORIZE_KEY, "false",
- QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, "false"
+ final QueryContext nonvector = new QueryContext(
+ ImmutableMap.of(
+ QueryContexts.VECTORIZE_KEY, "false",
+ QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, "false"
+ )
);
try (
- final DruidPlanner vectorPlanner = plannerFactory.createPlannerForTesting(vector, query);
- final DruidPlanner nonVectorPlanner = plannerFactory.createPlannerForTesting(nonvector, query)
+ final DruidPlanner vectorPlanner = plannerFactory.createPlannerForTesting(ENGINE, query, vector);
+ final DruidPlanner nonVectorPlanner = plannerFactory.createPlannerForTesting(ENGINE, query, nonvector)
) {
final PlannerResult vectorPlan = vectorPlanner.plan();
final PlannerResult nonVectorPlan = nonVectorPlanner.plan();
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/TestInsertQueryMaker.java b/sql/src/test/java/org/apache/druid/sql/calcite/TestInsertQueryMaker.java
index 31b9c31631..2576149d76 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/TestInsertQueryMaker.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/TestInsertQueryMaker.java
@@ -20,16 +20,11 @@
package org.apache.druid.sql.calcite;
import com.google.common.collect.ImmutableList;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.runtime.Hook;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.java.util.common.guava.Sequences;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.rel.DruidQuery;
-import org.apache.druid.sql.calcite.run.QueryFeature;
import org.apache.druid.sql.calcite.run.QueryMaker;
/**
@@ -37,56 +32,18 @@ import org.apache.druid.sql.calcite.run.QueryMaker;
*/
public class TestInsertQueryMaker implements QueryMaker
{
- private final RelDataType resultType;
private final String targetDataSource;
private final RowSignature signature;
public TestInsertQueryMaker(
- final RelDataTypeFactory typeFactory,
final String targetDataSource,
final RowSignature signature
)
{
- this.resultType = typeFactory.createStructType(
- ImmutableList.of(
- typeFactory.createSqlType(SqlTypeName.VARCHAR),
- typeFactory.createSqlType(SqlTypeName.OTHER)
- ),
- ImmutableList.of("dataSource", "signature")
- );
this.targetDataSource = targetDataSource;
this.signature = signature;
}
- @Override
- public boolean feature(final QueryFeature feature)
- {
- switch (feature) {
- // INSERT queries should stick to groupBy, scan.
- case CAN_RUN_TIMESERIES:
- case CAN_RUN_TOPN:
- case CAN_RUN_TIME_BOUNDARY:
- return false;
-
- // INSERT uses external data.
- case CAN_READ_EXTERNAL_DATA:
- return true;
-
- // INSERT uses Scan + ORDER BY.
- case SCAN_CAN_ORDER_BY_NON_TIME:
- return true;
-
- default:
- throw new IAE("Unrecognized feature: %s", feature);
- }
- }
-
- @Override
- public RelDataType getResultType()
- {
- return resultType;
- }
-
@Override
public Sequence<Object[]> runQuery(final DruidQuery druidQuery)
{
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/TestQueryMakerFactory.java b/sql/src/test/java/org/apache/druid/sql/calcite/TestQueryMakerFactory.java
deleted file mode 100644
index b5e50236fd..0000000000
--- a/sql/src/test/java/org/apache/druid/sql/calcite/TestQueryMakerFactory.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.sql.calcite;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.calcite.rel.RelRoot;
-import org.apache.druid.segment.column.RowSignature;
-import org.apache.druid.server.QueryLifecycleFactory;
-import org.apache.druid.sql.calcite.planner.PlannerContext;
-import org.apache.druid.sql.calcite.run.NativeQueryMakerFactory;
-import org.apache.druid.sql.calcite.run.QueryMaker;
-import org.apache.druid.sql.calcite.run.QueryMakerFactory;
-import org.apache.druid.sql.calcite.table.RowSignatures;
-
-public class TestQueryMakerFactory implements QueryMakerFactory
-{
- private final QueryLifecycleFactory queryLifecycleFactory;
- private final ObjectMapper jsonMapper;
-
- public TestQueryMakerFactory(
- final QueryLifecycleFactory queryLifecycleFactory,
- final ObjectMapper jsonMapper
- )
- {
- this.queryLifecycleFactory = queryLifecycleFactory;
- this.jsonMapper = jsonMapper;
- }
-
- @Override
- public QueryMaker buildForSelect(RelRoot relRoot, PlannerContext plannerContext)
- {
- return new NativeQueryMakerFactory(queryLifecycleFactory, jsonMapper).buildForSelect(relRoot, plannerContext);
- }
-
- @Override
- public QueryMaker buildForInsert(String targetDataSource, RelRoot relRoot, PlannerContext plannerContext)
- {
- final RowSignature signature = RowSignatures.fromRelDataType(
- relRoot.validatedRowType.getFieldNames(),
- relRoot.validatedRowType
- );
-
- return new TestInsertQueryMaker(relRoot.rel.getCluster().getTypeFactory(), targetDataSource, signature);
- }
-}
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionTestHelper.java b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionTestHelper.java
index ce255fd414..539e585e5f 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionTestHelper.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionTestHelper.java
@@ -63,7 +63,6 @@ import org.joda.time.DateTimeZone;
import org.junit.Assert;
import javax.annotation.Nullable;
-
import java.math.BigDecimal;
import java.util.Arrays;
import java.util.Collections;
@@ -88,6 +87,7 @@ class ExpressionTestHelper
NamedViewSchema.NAME, new NamedViewSchema(EasyMock.createMock(ViewSchema.class))
)
),
+ null /* Don't need engine */,
new QueryContext()
);
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/external/ExternalTableScanRuleTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/external/ExternalTableScanRuleTest.java
index c30b2c0262..d536cbcbbb 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/external/ExternalTableScanRuleTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/external/ExternalTableScanRuleTest.java
@@ -23,12 +23,12 @@ import com.google.common.collect.ImmutableMap;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.tools.ValidationException;
import org.apache.druid.query.QueryContext;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.QuerySegmentWalker;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.sql.calcite.run.NativeSqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchema;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.schema.NamedDruidSchema;
@@ -42,8 +42,12 @@ import org.junit.Test;
public class ExternalTableScanRuleTest
{
@Test
- public void testMatchesWhenExternalScanUnsupported() throws ValidationException
+ public void testMatchesWhenExternalScanUnsupported()
{
+ final NativeSqlEngine engine = CalciteTests.createMockSqlEngine(
+ EasyMock.createMock(QuerySegmentWalker.class),
+ EasyMock.createMock(QueryRunnerFactoryConglomerate.class)
+ );
final PlannerContext plannerContext = PlannerContext.create(
"DUMMY", // The actual query isn't important for this test
CalciteTests.createOperatorTable(),
@@ -57,20 +61,17 @@ public class ExternalTableScanRuleTest
NamedViewSchema.NAME, new NamedViewSchema(EasyMock.createMock(ViewSchema.class))
)
),
+ engine,
new QueryContext()
);
plannerContext.setQueryMaker(
- CalciteTests.createMockQueryMakerFactory(
- EasyMock.createMock(QuerySegmentWalker.class),
- EasyMock.createMock(QueryRunnerFactoryConglomerate.class)
- )
- .buildForSelect(EasyMock.createMock(RelRoot.class), plannerContext)
+ engine.buildQueryMakerForSelect(EasyMock.createMock(RelRoot.class), plannerContext)
);
ExternalTableScanRule rule = new ExternalTableScanRule(plannerContext);
rule.matches(EasyMock.createMock(RelOptRuleCall.class));
Assert.assertEquals(
- "SQL query requires scanning external datasources that is not suported.",
+ "Cannot use 'EXTERN' with SQL engine 'native'.",
plannerContext.getPlanningError()
);
}
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModuleTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModuleTest.java
index 72278c19a8..115b7c0237 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModuleTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModuleTest.java
@@ -121,8 +121,8 @@ public class CalcitePlannerModuleTest extends CalciteTestBase
binder.bind(ExprMacroTable.class).toInstance(macroTable);
binder.bind(AuthorizerMapper.class).toInstance(authorizerMapper);
binder.bind(String.class).annotatedWith(DruidSchemaName.class).toInstance(DRUID_SCHEMA_NAME);
- binder.bind(Key.get(new TypeLiteral<Set<SqlAggregator>>(){})).toInstance(aggregators);
- binder.bind(Key.get(new TypeLiteral<Set<SqlOperatorConversion>>(){})).toInstance(operatorConversions);
+ binder.bind(Key.get(new TypeLiteral<Set<SqlAggregator>>() {})).toInstance(aggregators);
+ binder.bind(Key.get(new TypeLiteral<Set<SqlOperatorConversion>>() {})).toInstance(operatorConversions);
binder.bind(DruidSchemaCatalog.class).toInstance(rootSchema);
},
target,
@@ -173,6 +173,7 @@ public class CalcitePlannerModuleTest extends CalciteTestBase
new DefaultObjectMapper(),
injector.getInstance(PlannerConfig.class),
rootSchema,
+ null,
new QueryContext()
);
boolean containsCustomRule = injector.getInstance(CalciteRulesManager.class)
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
index 492287b84d..6d910384e0 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
@@ -96,6 +96,7 @@ public class DruidRexExecutorTest extends InitializedNullHandlingTest
NamedViewSchema.NAME, new NamedViewSchema(EasyMock.createMock(ViewSchema.class))
)
),
+ null /* Don't need an engine */,
new QueryContext()
);
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java
index 1a6477688e..6b051e8109 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java
@@ -114,11 +114,12 @@ import org.apache.druid.server.security.NoopEscalator;
import org.apache.druid.server.security.ResourceType;
import org.apache.druid.sql.SqlLifecycleManager;
import org.apache.druid.sql.SqlStatementFactory;
+import org.apache.druid.sql.SqlStatementFactoryFactory;
import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.planner.PlannerFactory;
-import org.apache.druid.sql.calcite.run.NativeQueryMakerFactory;
-import org.apache.druid.sql.calcite.run.QueryMakerFactory;
+import org.apache.druid.sql.calcite.run.NativeSqlEngine;
+import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchema;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.schema.DruidSchemaManager;
@@ -144,7 +145,6 @@ import org.joda.time.Duration;
import org.joda.time.chrono.ISOChronology;
import javax.annotation.Nullable;
-
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
@@ -758,12 +758,12 @@ public class CalciteTests
public static final DruidViewMacroFactory DRUID_VIEW_MACRO_FACTORY = new TestDruidViewMacroFactory();
- public static QueryMakerFactory createMockQueryMakerFactory(
+ public static NativeSqlEngine createMockSqlEngine(
final QuerySegmentWalker walker,
final QueryRunnerFactoryConglomerate conglomerate
)
{
- return new NativeQueryMakerFactory(createMockQueryLifecycleFactory(walker, conglomerate), getJsonMapper());
+ return new NativeSqlEngine(createMockQueryLifecycleFactory(walker, conglomerate), getJsonMapper());
}
public static QueryLifecycleFactory createMockQueryLifecycleFactory(
@@ -790,17 +790,21 @@ public class CalciteTests
);
}
- public static SqlStatementFactory createSqlLifecycleFactory(final PlannerFactory plannerFactory)
+ public static SqlStatementFactory createSqlStatementFactory(
+ final SqlEngine engine,
+ final PlannerFactory plannerFactory
+ )
{
- return createSqlLifecycleFactory(plannerFactory, new AuthConfig());
+ return createSqlStatementFactory(engine, plannerFactory, new AuthConfig());
}
- public static SqlStatementFactory createSqlLifecycleFactory(
+ public static SqlStatementFactory createSqlStatementFactory(
+ final SqlEngine engine,
final PlannerFactory plannerFactory,
final AuthConfig authConfig
)
{
- return new SqlStatementFactory(
+ return new SqlStatementFactoryFactory(
plannerFactory,
new ServiceEmitter("dummy", "dummy", new NoopEmitter()),
new NoopRequestLogger(),
@@ -808,7 +812,7 @@ public class CalciteTests
authConfig,
Suppliers.ofInstance(new DefaultQueryConfig(ImmutableMap.of())),
new SqlLifecycleManager()
- );
+ ).factorize(engine);
}
public static ObjectMapper getJsonMapper()
diff --git a/sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java b/sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java
index 640584e713..4b87c207f0 100644
--- a/sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java
@@ -73,7 +73,6 @@ import org.junit.runner.RunWith;
import javax.validation.Validation;
import javax.validation.Validator;
-
import java.util.Map;
import java.util.Properties;
diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlHttpModuleTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlHttpModuleTest.java
index 69e05c12cd..e105398998 100644
--- a/sql/src/test/java/org/apache/druid/sql/http/SqlHttpModuleTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/http/SqlHttpModuleTest.java
@@ -24,12 +24,17 @@ import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
+import com.google.inject.util.Providers;
import org.apache.druid.guice.DruidGuiceExtensions;
import org.apache.druid.guice.LifecycleModule;
import org.apache.druid.guice.annotations.JSR311Resource;
import org.apache.druid.guice.annotations.Json;
import org.apache.druid.server.security.AuthorizerMapper;
import org.apache.druid.sql.SqlStatementFactory;
+import org.apache.druid.sql.SqlStatementFactoryFactory;
+import org.apache.druid.sql.calcite.run.NativeSqlEngine;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
import org.easymock.EasyMockRunner;
import org.easymock.Mock;
import org.junit.Assert;
@@ -46,7 +51,7 @@ public class SqlHttpModuleTest
@Mock
private ObjectMapper jsonMpper;
@Mock
- private SqlStatementFactory sqlLifecycleFactory;
+ private SqlStatementFactoryFactory sqlStatementFactoryFactory;
private SqlHttpModule target;
private Injector injector;
@@ -54,14 +59,20 @@ public class SqlHttpModuleTest
@Before
public void setUp()
{
+ EasyMock.expect(sqlStatementFactoryFactory.factorize(EasyMock.capture(Capture.newInstance())))
+ .andReturn(EasyMock.mock(SqlStatementFactory.class))
+ .anyTimes();
+ EasyMock.replay(sqlStatementFactoryFactory);
+
target = new SqlHttpModule();
injector = Guice.createInjector(
new LifecycleModule(),
new DruidGuiceExtensions(),
binder -> {
binder.bind(ObjectMapper.class).annotatedWith(Json.class).toInstance(jsonMpper);
- binder.bind(SqlStatementFactory.class).toInstance(sqlLifecycleFactory);
+ binder.bind(SqlStatementFactoryFactory.class).toInstance(sqlStatementFactoryFactory);
binder.bind(AuthorizerMapper.class).toInstance(new AuthorizerMapper(Collections.emptyMap()));
+ binder.bind(NativeSqlEngine.class).toProvider(Providers.of(new NativeSqlEngine(null, null)));
},
target
);
@@ -80,7 +91,7 @@ public class SqlHttpModuleTest
public void testSqlResourceIsAvailableViaJersey()
{
Set<Class<?>> jerseyResourceClasses =
- injector.getInstance(Key.get(new TypeLiteral<Set<Class<?>>>(){}, JSR311Resource.class));
+ injector.getInstance(Key.get(new TypeLiteral<Set<Class<?>>>() {}, JSR311Resource.class));
Assert.assertEquals(1, jerseyResourceClasses.size());
Assert.assertEquals(SqlResource.class, jerseyResourceClasses.iterator().next());
}
diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java
index 5b5e0b4a46..6560562da2 100644
--- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java
@@ -23,7 +23,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
-import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -73,10 +72,13 @@ import org.apache.druid.server.security.ForbiddenException;
import org.apache.druid.server.security.ResourceAction;
import org.apache.druid.sql.DirectStatement;
import org.apache.druid.sql.HttpStatement;
+import org.apache.druid.sql.PreparedStatement;
import org.apache.druid.sql.SqlLifecycleManager;
import org.apache.druid.sql.SqlPlanningException.PlanningError;
+import org.apache.druid.sql.SqlQueryPlus;
import org.apache.druid.sql.SqlStatementFactory;
import org.apache.druid.sql.SqlToolbox;
+import org.apache.druid.sql.calcite.parser.DruidSqlInsert;
import org.apache.druid.sql.calcite.planner.CalciteRulesManager;
import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
import org.apache.druid.sql.calcite.planner.DruidPlanner;
@@ -85,12 +87,15 @@ import org.apache.druid.sql.calcite.planner.PlannerContext;
import org.apache.druid.sql.calcite.planner.PlannerFactory;
import org.apache.druid.sql.calcite.planner.PlannerResult;
import org.apache.druid.sql.calcite.planner.UnsupportedSQLQueryException;
+import org.apache.druid.sql.calcite.run.NativeSqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.util.CalciteTestBase;
import org.apache.druid.sql.calcite.util.CalciteTests;
import org.apache.druid.sql.calcite.util.QueryLogHook;
import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker;
import org.easymock.EasyMock;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.MatcherAssert;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -105,7 +110,6 @@ import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import javax.ws.rs.core.StreamingOutput;
-
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -153,7 +157,8 @@ public class SqlResourceTest extends CalciteTestBase
private HttpServletRequest req;
private ListeningExecutorService executorService;
private SqlLifecycleManager lifecycleManager;
- private SqlStatementFactory sqlLifecycleFactory;
+ private NativeSqlEngine engine;
+ private SqlStatementFactory sqlStatementFactory;
private CountDownLatch lifecycleAddLatch;
private final SettableSupplier<NonnullPair<CountDownLatch, Boolean>> validateAndAuthorizeLatchSupplier = new SettableSupplier<>();
@@ -225,7 +230,6 @@ public class SqlResourceTest extends CalciteTestBase
final PlannerFactory plannerFactory = new PlannerFactory(
rootSchema,
- CalciteTests.createMockQueryMakerFactory(walker, conglomerate),
operatorTable,
macroTable,
plannerConfig,
@@ -249,15 +253,18 @@ public class SqlResourceTest extends CalciteTestBase
final ServiceEmitter emitter = new NoopServiceEmitter();
final AuthConfig authConfig = new AuthConfig();
final DefaultQueryConfig defaultQueryConfig = new DefaultQueryConfig(ImmutableMap.of());
- sqlLifecycleFactory = new SqlStatementFactory(
+ engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
+ final SqlToolbox sqlToolbox = new SqlToolbox(
+ engine,
plannerFactory,
emitter,
testRequestLogger,
scheduler,
authConfig,
- Suppliers.ofInstance(defaultQueryConfig),
+ defaultQueryConfig,
lifecycleManager
- )
+ );
+ sqlStatementFactory = new SqlStatementFactory()
{
@Override
public HttpStatement httpStatement(
@@ -266,7 +273,7 @@ public class SqlResourceTest extends CalciteTestBase
)
{
TestHttpStatement stmt = new TestHttpStatement(
- lifecycleToolbox,
+ sqlToolbox,
sqlQuery,
req,
validateAndAuthorizeLatchSupplier,
@@ -278,11 +285,23 @@ public class SqlResourceTest extends CalciteTestBase
onExecute = NULL_ACTION;
return stmt;
}
+
+ @Override
+ public DirectStatement directStatement(SqlQueryPlus sqlRequest)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public PreparedStatement preparedStatement(SqlQueryPlus sqlRequest)
+ {
+ throw new UnsupportedOperationException();
+ }
};
resource = new SqlResource(
JSON_MAPPER,
CalciteTests.TEST_AUTHORIZER_MAPPER,
- sqlLifecycleFactory,
+ sqlStatementFactory,
lifecycleManager,
new ServerConfig()
);
@@ -667,7 +686,7 @@ public class SqlResourceTest extends CalciteTestBase
public void testArrayResultFormatWithHeader_nullColumnType() throws Exception
{
// Test a query that returns null header for some of the columns
- final String query = "SELECT (1, 2)";
+ final String query = "SELECT (1, 2) FROM INFORMATION_SCHEMA.COLUMNS LIMIT 1";
Assert.assertEquals(
ImmutableList.of(
Collections.singletonList("EXPR$0"),
@@ -783,7 +802,7 @@ public class SqlResourceTest extends CalciteTestBase
@Test
public void testArrayLinesResultFormatWithHeader_nullColumnType() throws Exception
{
- final String query = "SELECT (1, 2)";
+ final String query = "SELECT (1, 2) FROM INFORMATION_SCHEMA.COLUMNS LIMIT 1";
final Pair<QueryException, String> pair = doPostRaw(
new SqlQuery(query, ResultFormat.ARRAYLINES, true, true, true, null, null)
);
@@ -1040,7 +1059,7 @@ public class SqlResourceTest extends CalciteTestBase
@Test
public void testObjectLinesResultFormatWithFullHeader_nullColumnType() throws Exception
{
- final String query = "SELECT (1, 2)";
+ final String query = "SELECT (1, 2) FROM INFORMATION_SCHEMA.COLUMNS LIMIT 1";
final Pair<QueryException, String> pair =
doPostRaw(new SqlQuery(query, ResultFormat.OBJECTLINES, true, true, true, null, null));
Assert.assertNull(pair.lhs);
@@ -1116,7 +1135,7 @@ public class SqlResourceTest extends CalciteTestBase
@Test
public void testCsvResultFormatWithHeaders_nullColumnType() throws Exception
{
- final String query = "SELECT (1, 2)";
+ final String query = "SELECT (1, 2) FROM INFORMATION_SCHEMA.COLUMNS LIMIT 1";
final Pair<QueryException, String> pair = doPostRaw(
new SqlQuery(query, ResultFormat.CSV, true, true, true, null, null)
);
@@ -1209,8 +1228,8 @@ public class SqlResourceTest extends CalciteTestBase
Assert.assertEquals(PlanningError.UNSUPPORTED_SQL_ERROR.getErrorClass(), exception.getErrorClass());
Assert.assertTrue(
exception.getMessage()
- .contains("Cannot build plan for query: SELECT dim1 FROM druid.foo ORDER BY dim1. " +
- "Possible error: SQL query requires order by non-time column [dim1 ASC] that is not supported.")
+ .contains("Cannot build plan for query. " +
+ "Possible error: SQL query requires order by non-time column [dim1 ASC] that is not supported.")
);
checkSqlRequestLog(false);
Assert.assertTrue(lifecycleManager.getAll("id").isEmpty());
@@ -1234,8 +1253,8 @@ public class SqlResourceTest extends CalciteTestBase
Assert.assertEquals(PlanningError.UNSUPPORTED_SQL_ERROR.getErrorClass(), exception.getErrorClass());
Assert.assertTrue(
exception.getMessage()
- .contains("Cannot build plan for query: SELECT max(dim1) FROM druid.foo. " +
- "Possible error: Max aggregation is not supported for 'STRING' type")
+ .contains("Cannot build plan for query. " +
+ "Possible error: Max aggregation is not supported for 'STRING' type")
);
checkSqlRequestLog(false);
Assert.assertTrue(lifecycleManager.getAll("id").isEmpty());
@@ -1346,7 +1365,7 @@ public class SqlResourceTest extends CalciteTestBase
resource = new SqlResource(
JSON_MAPPER,
CalciteTests.TEST_AUTHORIZER_MAPPER,
- sqlLifecycleFactory,
+ sqlStatementFactory,
lifecycleManager,
new ServerConfig() {
@Override
@@ -1391,7 +1410,7 @@ public class SqlResourceTest extends CalciteTestBase
resource = new SqlResource(
JSON_MAPPER,
CalciteTests.TEST_AUTHORIZER_MAPPER,
- sqlLifecycleFactory,
+ sqlStatementFactory,
lifecycleManager,
new ServerConfig()
{
@@ -1492,7 +1511,12 @@ public class SqlResourceTest extends CalciteTestBase
public void testQueryTimeoutException() throws Exception
{
final String sqlQueryId = "timeoutTest";
- Map<String, Object> queryContext = ImmutableMap.of(QueryContexts.TIMEOUT_KEY, 1, BaseQuery.SQL_QUERY_ID, sqlQueryId);
+ Map<String, Object> queryContext = ImmutableMap.of(
+ QueryContexts.TIMEOUT_KEY,
+ 1,
+ BaseQuery.SQL_QUERY_ID,
+ sqlQueryId
+ );
final QueryException timeoutException = doPost(
new SqlQuery(
"SELECT CAST(__time AS DATE), dim1, dim2, dim3 FROM druid.foo GROUP by __time, dim1, dim2, dim3 ORDER BY dim2 DESC",
@@ -1626,7 +1650,12 @@ public class SqlResourceTest extends CalciteTestBase
public void testQueryContextException() throws Exception
{
final String sqlQueryId = "badQueryContextTimeout";
- Map<String, Object> queryContext = ImmutableMap.of(QueryContexts.TIMEOUT_KEY, "2000'", BaseQuery.SQL_QUERY_ID, sqlQueryId);
+ Map<String, Object> queryContext = ImmutableMap.of(
+ QueryContexts.TIMEOUT_KEY,
+ "2000'",
+ BaseQuery.SQL_QUERY_ID,
+ sqlQueryId
+ );
final QueryException queryContextException = doPost(
new SqlQuery(
"SELECT 1337",
@@ -1646,6 +1675,31 @@ public class SqlResourceTest extends CalciteTestBase
Assert.assertTrue(lifecycleManager.getAll(sqlQueryId).isEmpty());
}
+ @Test
+ public void testQueryContextKeyNotAllowed() throws Exception
+ {
+ Map<String, Object> queryContext = ImmutableMap.of(DruidSqlInsert.SQL_INSERT_SEGMENT_GRANULARITY, "all");
+ final QueryException queryContextException = doPost(
+ new SqlQuery(
+ "SELECT 1337",
+ ResultFormat.OBJECT,
+ false,
+ false,
+ false,
+ queryContext,
+ null
+ )
+ ).lhs;
+ Assert.assertNotNull(queryContextException);
+ Assert.assertEquals(PlanningError.VALIDATION_ERROR.getErrorCode(), queryContextException.getErrorCode());
+ MatcherAssert.assertThat(
+ queryContextException.getMessage(),
+ CoreMatchers.containsString(
+ "Cannot execute query with context parameter [sqlInsertSegmentGranularity]")
+ );
+ checkSqlRequestLog(false);
+ }
+
@SuppressWarnings("unchecked")
private void checkSqlRequestLog(boolean success)
{
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org