You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/06/29 05:48:12 UTC
[05/50] kylin git commit: KYLIN-2671 Speed up prepared query execution
KYLIN-2671 Speed up prepared query execution
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56e9fb9e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56e9fb9e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56e9fb9e
Branch: refs/heads/master
Commit: 56e9fb9e9fa45e9bfb93dd1f4fd0a1027b539c7a
Parents: 1b58373
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Jun 15 17:52:22 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 15 18:18:45 2017 +0800
----------------------------------------------------------------------
.../calcite/prepare/CalcitePrepareImpl.java | 1500 ++++++++++++++++++
.../prepare/OnlyPrepareEarlyAbortException.java | 40 +
.../kylin/rest/controller/QueryController.java | 9 +-
.../apache/kylin/rest/service/QueryService.java | 108 +-
4 files changed, 1623 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/56e9fb9e/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
new file mode 100644
index 0000000..50939b3
--- /dev/null
+++ b/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
@@ -0,0 +1,1500 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.calcite.prepare;
+
+import org.apache.calcite.DataContext;
+import org.apache.calcite.adapter.enumerable.EnumerableBindable;
+import org.apache.calcite.adapter.enumerable.EnumerableCalc;
+import org.apache.calcite.adapter.enumerable.EnumerableConvention;
+import org.apache.calcite.adapter.enumerable.EnumerableInterpretable;
+import org.apache.calcite.adapter.enumerable.EnumerableInterpreterRule;
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.adapter.enumerable.EnumerableRules;
+import org.apache.calcite.adapter.enumerable.RexToLixTranslator;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.avatica.AvaticaParameter;
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.calcite.avatica.Meta;
+import org.apache.calcite.config.CalciteConnectionConfig;
+import org.apache.calcite.interpreter.BindableConvention;
+import org.apache.calcite.interpreter.Bindables;
+import org.apache.calcite.interpreter.Interpreters;
+import org.apache.calcite.jdbc.CalcitePrepare;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.jdbc.CalciteSchema.LatticeEntry;
+import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.linq4j.Linq4j;
+import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.linq4j.Queryable;
+import org.apache.calcite.linq4j.function.Function1;
+import org.apache.calcite.linq4j.tree.BinaryExpression;
+import org.apache.calcite.linq4j.tree.BlockStatement;
+import org.apache.calcite.linq4j.tree.Blocks;
+import org.apache.calcite.linq4j.tree.ConstantExpression;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.linq4j.tree.MemberExpression;
+import org.apache.calcite.linq4j.tree.MethodCallExpression;
+import org.apache.calcite.linq4j.tree.NewExpression;
+import org.apache.calcite.linq4j.tree.ParameterExpression;
+import org.apache.calcite.materialize.MaterializationService;
+import org.apache.calcite.plan.Contexts;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCostFactory;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.plan.volcano.VolcanoPlanner;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelCollations;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule;
+import org.apache.calcite.rel.rules.AggregateReduceFunctionsRule;
+import org.apache.calcite.rel.rules.AggregateStarTableRule;
+import org.apache.calcite.rel.rules.AggregateValuesRule;
+import org.apache.calcite.rel.rules.FilterAggregateTransposeRule;
+import org.apache.calcite.rel.rules.FilterJoinRule;
+import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
+import org.apache.calcite.rel.rules.FilterTableScanRule;
+import org.apache.calcite.rel.rules.JoinAssociateRule;
+import org.apache.calcite.rel.rules.JoinCommuteRule;
+import org.apache.calcite.rel.rules.JoinPushExpressionsRule;
+import org.apache.calcite.rel.rules.JoinPushThroughJoinRule;
+import org.apache.calcite.rel.rules.MaterializedViewFilterScanRule;
+import org.apache.calcite.rel.rules.ProjectFilterTransposeRule;
+import org.apache.calcite.rel.rules.ProjectMergeRule;
+import org.apache.calcite.rel.rules.ProjectTableScanRule;
+import org.apache.calcite.rel.rules.ProjectWindowTransposeRule;
+import org.apache.calcite.rel.rules.ReduceExpressionsRule;
+import org.apache.calcite.rel.rules.SortJoinTransposeRule;
+import org.apache.calcite.rel.rules.SortProjectTransposeRule;
+import org.apache.calcite.rel.rules.SortUnionTransposeRule;
+import org.apache.calcite.rel.rules.TableScanRule;
+import org.apache.calcite.rel.rules.ValuesReduceRule;
+import org.apache.calcite.rel.stream.StreamRules;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexProgram;
+import org.apache.calcite.runtime.Bindable;
+import org.apache.calcite.runtime.Hook;
+import org.apache.calcite.runtime.Typed;
+import org.apache.calcite.schema.Schemas;
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.server.CalciteServerStatement;
+import org.apache.calcite.sql.SqlBinaryOperator;
+import org.apache.calcite.sql.SqlExecutableStatement;
+import org.apache.calcite.sql.SqlExplainFormat;
+import org.apache.calcite.sql.SqlExplainLevel;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.SqlUtil;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.parser.SqlParserImplFactory;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.util.ChainedSqlOperatorTable;
+import org.apache.calcite.sql.validate.SqlConformance;
+import org.apache.calcite.sql.validate.SqlValidator;
+import org.apache.calcite.sql2rel.SqlRexConvertletTable;
+import org.apache.calcite.sql2rel.SqlToRelConverter;
+import org.apache.calcite.sql2rel.StandardConvertletTable;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.util.ImmutableIntList;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Util;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.sql.DatabaseMetaData;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.calcite.util.Static.RESOURCE;
+
+/*
+ * OVERRIDE POINT:
+ * - grep KYLIN_ONLY_PREPARE
+ */
+/**
+ * Shit just got real.
+ *
+ * <p>This class is public so that projects that create their own JDBC driver
+ * and server can fine-tune preferences. However, this class and its methods are
+ * subject to change without notice.</p>
+ */
+public class CalcitePrepareImpl implements CalcitePrepare {
+
+ public static final ThreadLocal<Boolean> KYLIN_ONLY_PREPARE = new ThreadLocal<>();
+
+ public static final boolean DEBUG = Util.getBooleanProperty("calcite.debug");
+
+ public static final boolean COMMUTE =
+ Util.getBooleanProperty("calcite.enable.join.commute");
+
+ /** Whether to enable the collation trait. Some extra optimizations are
+ * possible if enabled, but queries should work either way. At some point
+ * this will become a preference, or we will run multiple phases: first
+ * disabled, then enabled. */
+ private static final boolean ENABLE_COLLATION_TRAIT = true;
+
+ /** Whether the bindable convention should be the root convention of any
+ * plan. If not, enumerable convention is the default. */
+ public final boolean enableBindable = Hook.ENABLE_BINDABLE.get(false);
+
+ /** Whether the enumerable convention is enabled. */
+ public static final boolean ENABLE_ENUMERABLE = true;
+
+ /** Whether the streaming is enabled. */
+ public static final boolean ENABLE_STREAM = true;
+
+ private static final Set<String> SIMPLE_SQLS =
+ ImmutableSet.of(
+ "SELECT 1",
+ "select 1",
+ "SELECT 1 FROM DUAL",
+ "select 1 from dual",
+ "values 1",
+ "VALUES 1");
+
+ public static final List<RelOptRule> ENUMERABLE_RULES =
+ ImmutableList.of(
+ EnumerableRules.ENUMERABLE_JOIN_RULE,
+ EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE,
+ EnumerableRules.ENUMERABLE_SEMI_JOIN_RULE,
+ EnumerableRules.ENUMERABLE_CORRELATE_RULE,
+ EnumerableRules.ENUMERABLE_PROJECT_RULE,
+ EnumerableRules.ENUMERABLE_FILTER_RULE,
+ EnumerableRules.ENUMERABLE_AGGREGATE_RULE,
+ EnumerableRules.ENUMERABLE_SORT_RULE,
+ EnumerableRules.ENUMERABLE_LIMIT_RULE,
+ EnumerableRules.ENUMERABLE_COLLECT_RULE,
+ EnumerableRules.ENUMERABLE_UNCOLLECT_RULE,
+ EnumerableRules.ENUMERABLE_UNION_RULE,
+ EnumerableRules.ENUMERABLE_INTERSECT_RULE,
+ EnumerableRules.ENUMERABLE_MINUS_RULE,
+ EnumerableRules.ENUMERABLE_TABLE_MODIFICATION_RULE,
+ EnumerableRules.ENUMERABLE_VALUES_RULE,
+ EnumerableRules.ENUMERABLE_WINDOW_RULE,
+ EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE,
+ EnumerableRules.ENUMERABLE_TABLE_FUNCTION_SCAN_RULE);
+
+ private static final List<RelOptRule> DEFAULT_RULES =
+ ImmutableList.of(
+ AggregateStarTableRule.INSTANCE,
+ AggregateStarTableRule.INSTANCE2,
+ TableScanRule.INSTANCE,
+ COMMUTE
+ ? JoinAssociateRule.INSTANCE
+ : ProjectMergeRule.INSTANCE,
+ FilterTableScanRule.INSTANCE,
+ ProjectFilterTransposeRule.INSTANCE,
+ FilterProjectTransposeRule.INSTANCE,
+ FilterJoinRule.FILTER_ON_JOIN,
+ JoinPushExpressionsRule.INSTANCE,
+ AggregateExpandDistinctAggregatesRule.INSTANCE,
+ AggregateReduceFunctionsRule.INSTANCE,
+ FilterAggregateTransposeRule.INSTANCE,
+ ProjectWindowTransposeRule.INSTANCE,
+ JoinCommuteRule.INSTANCE,
+ JoinPushThroughJoinRule.RIGHT,
+ JoinPushThroughJoinRule.LEFT,
+ SortProjectTransposeRule.INSTANCE,
+ SortJoinTransposeRule.INSTANCE,
+ SortUnionTransposeRule.INSTANCE);
+
+ private static final List<RelOptRule> CONSTANT_REDUCTION_RULES =
+ ImmutableList.of(
+ ReduceExpressionsRule.PROJECT_INSTANCE,
+ ReduceExpressionsRule.FILTER_INSTANCE,
+ ReduceExpressionsRule.CALC_INSTANCE,
+ ReduceExpressionsRule.JOIN_INSTANCE,
+ ValuesReduceRule.FILTER_INSTANCE,
+ ValuesReduceRule.PROJECT_FILTER_INSTANCE,
+ ValuesReduceRule.PROJECT_INSTANCE,
+ AggregateValuesRule.INSTANCE);
+
+ public CalcitePrepareImpl() {
+ }
+
+ public ParseResult parse(
+ Context context, String sql) {
+ return parse_(context, sql, false, false, false);
+ }
+
+ public ConvertResult convert(Context context, String sql) {
+ return (ConvertResult) parse_(context, sql, true, false, false);
+ }
+
+ public AnalyzeViewResult analyzeView(Context context, String sql, boolean fail) {
+ return (AnalyzeViewResult) parse_(context, sql, true, true, fail);
+ }
+
+ /** Shared implementation for {@link #parse}, {@link #convert} and
+ * {@link #analyzeView}. */
+ private ParseResult parse_(Context context, String sql, boolean convert,
+ boolean analyze, boolean fail) {
+ final JavaTypeFactory typeFactory = context.getTypeFactory();
+ CalciteCatalogReader catalogReader =
+ new CalciteCatalogReader(
+ context.getRootSchema(),
+ context.config().caseSensitive(),
+ context.getDefaultSchemaPath(),
+ typeFactory);
+ SqlParser parser = createParser(sql);
+ SqlNode sqlNode;
+ try {
+ sqlNode = parser.parseStmt();
+ } catch (SqlParseException e) {
+ throw new RuntimeException("parse failed", e);
+ }
+ final SqlValidator validator = createSqlValidator(context, catalogReader);
+ SqlNode sqlNode1 = validator.validate(sqlNode);
+ if (convert) {
+ return convert_(
+ context, sql, analyze, fail, catalogReader, validator, sqlNode1);
+ }
+ return new ParseResult(this, validator, sql, sqlNode1,
+ validator.getValidatedNodeType(sqlNode1));
+ }
+
+ private ParseResult convert_(Context context, String sql, boolean analyze,
+ boolean fail, CalciteCatalogReader catalogReader, SqlValidator validator,
+ SqlNode sqlNode1) {
+ final JavaTypeFactory typeFactory = context.getTypeFactory();
+ final Convention resultConvention =
+ enableBindable ? BindableConvention.INSTANCE
+ : EnumerableConvention.INSTANCE;
+ final HepPlanner planner = new HepPlanner(new HepProgramBuilder().build());
+ planner.addRelTraitDef(ConventionTraitDef.INSTANCE);
+
+ final SqlToRelConverter.ConfigBuilder configBuilder =
+ SqlToRelConverter.configBuilder().withTrimUnusedFields(true);
+ if (analyze) {
+ configBuilder.withConvertTableAccess(false);
+ }
+
+ final CalcitePreparingStmt preparingStmt =
+ new CalcitePreparingStmt(this, context, catalogReader, typeFactory,
+ context.getRootSchema(), null, planner, resultConvention,
+ createConvertletTable());
+ final SqlToRelConverter converter =
+ preparingStmt.getSqlToRelConverter(validator, catalogReader,
+ configBuilder.build());
+
+ final RelRoot root = converter.convertQuery(sqlNode1, false, true);
+ if (analyze) {
+ return analyze_(validator, sql, sqlNode1, root, fail);
+ }
+ return new ConvertResult(this, validator, sql, sqlNode1,
+ validator.getValidatedNodeType(sqlNode1), root);
+ }
+
+ private AnalyzeViewResult analyze_(SqlValidator validator, String sql,
+ SqlNode sqlNode, RelRoot root, boolean fail) {
+ final RexBuilder rexBuilder = root.rel.getCluster().getRexBuilder();
+ RelNode rel = root.rel;
+ final RelNode viewRel = rel;
+ Project project;
+ if (rel instanceof Project) {
+ project = (Project) rel;
+ rel = project.getInput();
+ } else {
+ project = null;
+ }
+ Filter filter;
+ if (rel instanceof Filter) {
+ filter = (Filter) rel;
+ rel = filter.getInput();
+ } else {
+ filter = null;
+ }
+ TableScan scan;
+ if (rel instanceof TableScan) {
+ scan = (TableScan) rel;
+ } else {
+ scan = null;
+ }
+ if (scan == null) {
+ if (fail) {
+ throw validator.newValidationError(sqlNode,
+ RESOURCE.modifiableViewMustBeBasedOnSingleTable());
+ }
+ return new AnalyzeViewResult(this, validator, sql, sqlNode,
+ validator.getValidatedNodeType(sqlNode), root, null, null, null,
+ null, false);
+ }
+ final RelOptTable targetRelTable = scan.getTable();
+ final RelDataType targetRowType = targetRelTable.getRowType();
+ final Table table = targetRelTable.unwrap(Table.class);
+ final List<String> tablePath = targetRelTable.getQualifiedName();
+ assert table != null;
+ List<Integer> columnMapping;
+ final Map<Integer, RexNode> projectMap = new HashMap<>();
+ if (project == null) {
+ columnMapping = ImmutableIntList.range(0, targetRowType.getFieldCount());
+ } else {
+ columnMapping = new ArrayList<>();
+ for (Ord<RexNode> node : Ord.zip(project.getProjects())) {
+ if (node.e instanceof RexInputRef) {
+ RexInputRef rexInputRef = (RexInputRef) node.e;
+ int index = rexInputRef.getIndex();
+ if (projectMap.get(index) != null) {
+ if (fail) {
+ throw validator.newValidationError(sqlNode,
+ RESOURCE.moreThanOneMappedColumn(
+ targetRowType.getFieldList().get(index).getName(),
+ Util.last(tablePath)));
+ }
+ return new AnalyzeViewResult(this, validator, sql, sqlNode,
+ validator.getValidatedNodeType(sqlNode), root, null, null, null,
+ null, false);
+ }
+ projectMap.put(index, rexBuilder.makeInputRef(viewRel, node.i));
+ columnMapping.add(index);
+ } else {
+ columnMapping.add(-1);
+ }
+ }
+ }
+ final RexNode constraint;
+ if (filter != null) {
+ constraint = filter.getCondition();
+ } else {
+ constraint = rexBuilder.makeLiteral(true);
+ }
+ final List<RexNode> filters = new ArrayList<>();
+ // If we put a constraint in projectMap above, then filters will not be empty despite
+ // being a modifiable view.
+ final List<RexNode> filters2 = new ArrayList<>();
+ boolean retry = false;
+ RelOptUtil.inferViewPredicates(projectMap, filters, constraint);
+ if (fail && !filters.isEmpty()) {
+ final Map<Integer, RexNode> projectMap2 = new HashMap<>();
+ RelOptUtil.inferViewPredicates(projectMap2, filters2, constraint);
+ if (!filters2.isEmpty()) {
+ throw validator.newValidationError(sqlNode,
+ RESOURCE.modifiableViewMustHaveOnlyEqualityPredicates());
+ }
+ retry = true;
+ }
+
+ // Check that all columns that are not projected have a constant value
+ for (RelDataTypeField field : targetRowType.getFieldList()) {
+ final int x = columnMapping.indexOf(field.getIndex());
+ if (x >= 0) {
+ assert Util.skip(columnMapping, x + 1).indexOf(field.getIndex()) < 0
+ : "column projected more than once; should have checked above";
+ continue; // target column is projected
+ }
+ if (projectMap.get(field.getIndex()) != null) {
+ continue; // constant expression
+ }
+ if (field.getType().isNullable()) {
+ continue; // don't need expression for nullable columns; NULL suffices
+ }
+ if (fail) {
+ throw validator.newValidationError(sqlNode,
+ RESOURCE.noValueSuppliedForViewColumn(field.getName(),
+ Util.last(tablePath)));
+ }
+ return new AnalyzeViewResult(this, validator, sql, sqlNode,
+ validator.getValidatedNodeType(sqlNode), root, null, null, null,
+ null, false);
+ }
+
+ final boolean modifiable = filters.isEmpty() || retry && filters2.isEmpty();
+ return new AnalyzeViewResult(this, validator, sql, sqlNode,
+ validator.getValidatedNodeType(sqlNode), root, modifiable ? table : null,
+ ImmutableList.copyOf(tablePath),
+ constraint, ImmutableIntList.copyOf(columnMapping),
+ modifiable);
+ }
+
+ @Override public void executeDdl(Context context, SqlNode node) {
+ if (node instanceof SqlExecutableStatement) {
+ SqlExecutableStatement statement = (SqlExecutableStatement) node;
+ statement.execute(context);
+ return;
+ }
+ throw new UnsupportedOperationException();
+ }
+
+ /** Factory method for default SQL parser. */
+ protected SqlParser createParser(String sql) {
+ return createParser(sql, createParserConfig());
+ }
+
+ /** Factory method for SQL parser with a given configuration. */
+ protected SqlParser createParser(String sql,
+ SqlParser.ConfigBuilder parserConfig) {
+ return SqlParser.create(sql, parserConfig.build());
+ }
+
+ /** Factory method for SQL parser configuration. */
+ protected SqlParser.ConfigBuilder createParserConfig() {
+ return SqlParser.configBuilder();
+ }
+
+ /** Factory method for default convertlet table. */
+ protected SqlRexConvertletTable createConvertletTable() {
+ return StandardConvertletTable.INSTANCE;
+ }
+
+ /** Factory method for cluster. */
+ protected RelOptCluster createCluster(RelOptPlanner planner,
+ RexBuilder rexBuilder) {
+ return RelOptCluster.create(planner, rexBuilder);
+ }
+
+ /** Creates a collection of planner factories.
+ *
+ * <p>The collection must have at least one factory, and each factory must
+ * create a planner. If the collection has more than one planner, Calcite will
+ * try each planner in turn.</p>
+ *
+ * <p>One of the things you can do with this mechanism is to try a simpler,
+ * faster, planner with a smaller rule set first, then fall back to a more
+ * complex planner for complex and costly queries.</p>
+ *
+ * <p>The default implementation returns a factory that calls
+ * {@link #createPlanner(org.apache.calcite.jdbc.CalcitePrepare.Context)}.</p>
+ */
+ protected List<Function1<Context, RelOptPlanner>> createPlannerFactories() {
+ return Collections.<Function1<Context, RelOptPlanner>>singletonList(
+ new Function1<Context, RelOptPlanner>() {
+ public RelOptPlanner apply(Context context) {
+ return createPlanner(context, null, null);
+ }
+ });
+ }
+
+ /** Creates a query planner and initializes it with a default set of
+ * rules. */
+ protected RelOptPlanner createPlanner(CalcitePrepare.Context prepareContext) {
+ return createPlanner(prepareContext, null, null);
+ }
+
+ /** Creates a query planner and initializes it with a default set of
+ * rules. */
+ protected RelOptPlanner createPlanner(
+ final CalcitePrepare.Context prepareContext,
+ org.apache.calcite.plan.Context externalContext,
+ RelOptCostFactory costFactory) {
+ if (externalContext == null) {
+ externalContext = Contexts.of(prepareContext.config());
+ }
+ final VolcanoPlanner planner =
+ new VolcanoPlanner(costFactory, externalContext);
+ planner.addRelTraitDef(ConventionTraitDef.INSTANCE);
+ if (ENABLE_COLLATION_TRAIT) {
+ planner.addRelTraitDef(RelCollationTraitDef.INSTANCE);
+ planner.registerAbstractRelationalRules();
+ }
+ RelOptUtil.registerAbstractRels(planner);
+ for (RelOptRule rule : DEFAULT_RULES) {
+ planner.addRule(rule);
+ }
+ if (prepareContext.config().materializationsEnabled()) {
+ planner.addRule(MaterializedViewFilterScanRule.INSTANCE);
+ }
+ if (enableBindable) {
+ for (RelOptRule rule : Bindables.RULES) {
+ planner.addRule(rule);
+ }
+ }
+ planner.addRule(Bindables.BINDABLE_TABLE_SCAN_RULE);
+ planner.addRule(ProjectTableScanRule.INSTANCE);
+ planner.addRule(ProjectTableScanRule.INTERPRETER);
+
+ if (ENABLE_ENUMERABLE) {
+ for (RelOptRule rule : ENUMERABLE_RULES) {
+ planner.addRule(rule);
+ }
+ planner.addRule(EnumerableInterpreterRule.INSTANCE);
+ }
+
+ if (enableBindable && ENABLE_ENUMERABLE) {
+ planner.addRule(
+ EnumerableBindable.EnumerableToBindableConverterRule.INSTANCE);
+ }
+
+ if (ENABLE_STREAM) {
+ for (RelOptRule rule : StreamRules.RULES) {
+ planner.addRule(rule);
+ }
+ }
+
+ // Change the below to enable constant-reduction.
+ if (false) {
+ for (RelOptRule rule : CONSTANT_REDUCTION_RULES) {
+ planner.addRule(rule);
+ }
+ }
+
+ final SparkHandler spark = prepareContext.spark();
+ if (spark.enabled()) {
+ spark.registerRules(
+ new SparkHandler.RuleSetBuilder() {
+ public void addRule(RelOptRule rule) {
+ // TODO:
+ }
+
+ public void removeRule(RelOptRule rule) {
+ // TODO:
+ }
+ });
+ }
+
+ Hook.PLANNER.run(planner); // allow test to add or remove rules
+
+ return planner;
+ }
+
+ public <T> CalciteSignature<T> prepareQueryable(
+ Context context,
+ Queryable<T> queryable) {
+ return prepare_(context, Query.of(queryable), queryable.getElementType(),
+ -1);
+ }
+
+ public <T> CalciteSignature<T> prepareSql(
+ Context context,
+ Query<T> query,
+ Type elementType,
+ long maxRowCount) {
+ return prepare_(context, query, elementType, maxRowCount);
+ }
+
+ <T> CalciteSignature<T> prepare_(
+ Context context,
+ Query<T> query,
+ Type elementType,
+ long maxRowCount) {
+ if (SIMPLE_SQLS.contains(query.sql)) {
+ return simplePrepare(context, query.sql);
+ }
+
+ if(KYLIN_ONLY_PREPARE.get() != null && KYLIN_ONLY_PREPARE.get()) {
+ ParseResult parseResult = parse(context, query.sql);
+ Class<OnlyPrepareEarlyAbortException> onlyPrepareEarlyAbortExceptionClass =
+ OnlyPrepareEarlyAbortException.class;
+ throw new OnlyPrepareEarlyAbortException(context, parseResult);
+ }
+
+ final JavaTypeFactory typeFactory = context.getTypeFactory();
+ CalciteCatalogReader catalogReader =
+ new CalciteCatalogReader(
+ context.getRootSchema(),
+ context.config().caseSensitive(),
+ context.getDefaultSchemaPath(),
+ typeFactory);
+ final List<Function1<Context, RelOptPlanner>> plannerFactories =
+ createPlannerFactories();
+ if (plannerFactories.isEmpty()) {
+ throw new AssertionError("no planner factories");
+ }
+ RuntimeException exception = Util.FoundOne.NULL;
+ for (Function1<Context, RelOptPlanner> plannerFactory : plannerFactories) {
+ final RelOptPlanner planner = plannerFactory.apply(context);
+ if (planner == null) {
+ throw new AssertionError("factory returned null planner");
+ }
+ try {
+ return prepare2_(context, query, elementType, maxRowCount,
+ catalogReader, planner);
+ } catch (RelOptPlanner.CannotPlanException e) {
+ exception = e;
+ }
+ }
+ throw exception;
+ }
+
+ /** Quickly prepares a simple SQL statement, circumventing the usual
+ * preparation process. */
+ private <T> CalciteSignature<T> simplePrepare(Context context, String sql) {
+ final JavaTypeFactory typeFactory = context.getTypeFactory();
+ final RelDataType x =
+ typeFactory.builder()
+ .add(SqlUtil.deriveAliasFromOrdinal(0), SqlTypeName.INTEGER)
+ .build();
+ @SuppressWarnings("unchecked")
+ final List<T> list = (List) ImmutableList.of(1);
+ final List<String> origin = null;
+ final List<List<String>> origins =
+ Collections.nCopies(x.getFieldCount(), origin);
+ final List<ColumnMetaData> columns =
+ getColumnMetaDataList(typeFactory, x, x, origins);
+ final Meta.CursorFactory cursorFactory =
+ Meta.CursorFactory.deduce(columns, null);
+ return new CalciteSignature<>(
+ sql,
+ ImmutableList.<AvaticaParameter>of(),
+ ImmutableMap.<String, Object>of(),
+ x,
+ columns,
+ cursorFactory,
+ ImmutableList.<RelCollation>of(),
+ -1,
+ new Bindable<T>() {
+ public Enumerable<T> bind(DataContext dataContext) {
+ return Linq4j.asEnumerable(list);
+ }
+ },
+ Meta.StatementType.SELECT);
+ }
+
+ /**
+ * Deduces the broad type of statement.
+ * Currently returns SELECT for most statement types, but this may change.
+ *
+ * @param kind Kind of statement
+ */
+ private Meta.StatementType getStatementType(SqlKind kind) {
+ switch (kind) {
+ case INSERT:
+ case DELETE:
+ case UPDATE:
+ return Meta.StatementType.IS_DML;
+ default:
+ return Meta.StatementType.SELECT;
+ }
+ }
+
+ /**
+ * Deduces the broad type of statement for a prepare result.
+ * Currently returns SELECT for most statement types, but this may change.
+ *
+ * @param preparedResult Prepare result
+ */
+ private Meta.StatementType getStatementType(Prepare.PreparedResult preparedResult) {
+ if (preparedResult.isDml()) {
+ return Meta.StatementType.IS_DML;
+ } else {
+ return Meta.StatementType.SELECT;
+ }
+ }
+
+ <T> CalciteSignature<T> prepare2_(
+ Context context,
+ Query<T> query,
+ Type elementType,
+ long maxRowCount,
+ CalciteCatalogReader catalogReader,
+ RelOptPlanner planner) {
+ final JavaTypeFactory typeFactory = context.getTypeFactory();
+ final EnumerableRel.Prefer prefer;
+ if (elementType == Object[].class) {
+ prefer = EnumerableRel.Prefer.ARRAY;
+ } else {
+ prefer = EnumerableRel.Prefer.CUSTOM;
+ }
+ final Convention resultConvention =
+ enableBindable ? BindableConvention.INSTANCE
+ : EnumerableConvention.INSTANCE;
+ final CalcitePreparingStmt preparingStmt =
+ new CalcitePreparingStmt(this, context, catalogReader, typeFactory,
+ context.getRootSchema(), prefer, planner, resultConvention,
+ createConvertletTable());
+
+ final RelDataType x;
+ final Prepare.PreparedResult preparedResult;
+ final Meta.StatementType statementType;
+ if (query.sql != null) {
+ final CalciteConnectionConfig config = context.config();
+ final SqlParser.ConfigBuilder parserConfig = createParserConfig()
+ .setQuotedCasing(config.quotedCasing())
+ .setUnquotedCasing(config.unquotedCasing())
+ .setQuoting(config.quoting())
+ .setConformance(config.conformance());
+ final SqlParserImplFactory parserFactory =
+ config.parserFactory(SqlParserImplFactory.class, null);
+ if (parserFactory != null) {
+ parserConfig.setParserFactory(parserFactory);
+ }
+ SqlParser parser = createParser(query.sql, parserConfig);
+ SqlNode sqlNode;
+ try {
+ sqlNode = parser.parseStmt();
+ statementType = getStatementType(sqlNode.getKind());
+ } catch (SqlParseException e) {
+ throw new RuntimeException(
+ "parse failed: " + e.getMessage(), e);
+ }
+
+ Hook.PARSE_TREE.run(new Object[] {query.sql, sqlNode});
+
+ if (sqlNode.getKind().belongsTo(SqlKind.DDL)) {
+ executeDdl(context, sqlNode);
+
+ // Return a dummy signature that contains no rows
+ final Bindable<T> bindable = new Bindable<T>() {
+ public Enumerable<T> bind(DataContext dataContext) {
+ return Linq4j.emptyEnumerable();
+ }
+ };
+ return new CalciteSignature<>(query.sql,
+ ImmutableList.<AvaticaParameter>of(),
+ ImmutableMap.<String, Object>of(), null,
+ ImmutableList.<ColumnMetaData>of(), Meta.CursorFactory.OBJECT,
+ ImmutableList.<RelCollation>of(), -1, bindable);
+ }
+
+ final SqlValidator validator =
+ createSqlValidator(context, catalogReader);
+ validator.setIdentifierExpansion(true);
+ validator.setDefaultNullCollation(config.defaultNullCollation());
+
+ preparedResult = preparingStmt.prepareSql(
+ sqlNode, Object.class, validator, true);
+ switch (sqlNode.getKind()) {
+ case INSERT:
+ case DELETE:
+ case UPDATE:
+ case EXPLAIN:
+ // FIXME: getValidatedNodeType is wrong for DML
+ x = RelOptUtil.createDmlRowType(sqlNode.getKind(), typeFactory);
+ break;
+ default:
+ x = validator.getValidatedNodeType(sqlNode);
+ }
+ } else if (query.queryable != null) {
+ x = context.getTypeFactory().createType(elementType);
+ preparedResult =
+ preparingStmt.prepareQueryable(query.queryable, x);
+ statementType = getStatementType(preparedResult);
+ } else {
+ assert query.rel != null;
+ x = query.rel.getRowType();
+ preparedResult = preparingStmt.prepareRel(query.rel);
+ statementType = getStatementType(preparedResult);
+ }
+
+ final List<AvaticaParameter> parameters = new ArrayList<>();
+ final RelDataType parameterRowType = preparedResult.getParameterRowType();
+ for (RelDataTypeField field : parameterRowType.getFieldList()) {
+ RelDataType type = field.getType();
+ parameters.add(
+ new AvaticaParameter(
+ false,
+ getPrecision(type),
+ getScale(type),
+ getTypeOrdinal(type),
+ getTypeName(type),
+ getClassName(type),
+ field.getName()));
+ }
+
+ RelDataType jdbcType = makeStruct(typeFactory, x);
+ final List<List<String>> originList = preparedResult.getFieldOrigins();
+ final List<ColumnMetaData> columns =
+ getColumnMetaDataList(typeFactory, x, jdbcType, originList);
+ Class resultClazz = null;
+ if (preparedResult instanceof Typed) {
+ resultClazz = (Class) ((Typed) preparedResult).getElementType();
+ }
+ final Meta.CursorFactory cursorFactory =
+ preparingStmt.resultConvention == BindableConvention.INSTANCE
+ ? Meta.CursorFactory.ARRAY
+ : Meta.CursorFactory.deduce(columns, resultClazz);
+ //noinspection unchecked
+ final Bindable<T> bindable = preparedResult.getBindable(cursorFactory);
+ return new CalciteSignature<>(
+ query.sql,
+ parameters,
+ preparingStmt.internalParameters,
+ jdbcType,
+ columns,
+ cursorFactory,
+ preparedResult instanceof Prepare.PreparedResultImpl
+ ? ((Prepare.PreparedResultImpl) preparedResult).collations
+ : ImmutableList.<RelCollation>of(),
+ maxRowCount,
+ bindable,
+ statementType);
+ }
+
+ private SqlValidator createSqlValidator(Context context,
+ CalciteCatalogReader catalogReader) {
+ final SqlOperatorTable opTab0 =
+ context.config().fun(SqlOperatorTable.class,
+ SqlStdOperatorTable.instance());
+ final SqlOperatorTable opTab =
+ ChainedSqlOperatorTable.of(opTab0, catalogReader);
+ final JavaTypeFactory typeFactory = context.getTypeFactory();
+ final SqlConformance conformance = context.config().conformance();
+ return new CalciteSqlValidator(opTab, catalogReader, typeFactory,
+ conformance);
+ }
+
+ private List<ColumnMetaData> getColumnMetaDataList(
+ JavaTypeFactory typeFactory, RelDataType x, RelDataType jdbcType,
+ List<List<String>> originList) {
+ final List<ColumnMetaData> columns = new ArrayList<>();
+ for (Ord<RelDataTypeField> pair : Ord.zip(jdbcType.getFieldList())) {
+ final RelDataTypeField field = pair.e;
+ final RelDataType type = field.getType();
+ final RelDataType fieldType =
+ x.isStruct() ? x.getFieldList().get(pair.i).getType() : type;
+ columns.add(
+ metaData(typeFactory, columns.size(), field.getName(), type,
+ fieldType, originList.get(pair.i)));
+ }
+ return columns;
+ }
+
+ private ColumnMetaData metaData(JavaTypeFactory typeFactory, int ordinal,
+ String fieldName, RelDataType type, RelDataType fieldType,
+ List<String> origins) {
+ final ColumnMetaData.AvaticaType avaticaType =
+ avaticaType(typeFactory, type, fieldType);
+ return new ColumnMetaData(
+ ordinal,
+ false,
+ true,
+ false,
+ false,
+ type.isNullable()
+ ? DatabaseMetaData.columnNullable
+ : DatabaseMetaData.columnNoNulls,
+ true,
+ type.getPrecision(),
+ fieldName,
+ origin(origins, 0),
+ origin(origins, 2),
+ getPrecision(type),
+ getScale(type),
+ origin(origins, 1),
+ null,
+ avaticaType,
+ true,
+ false,
+ false,
+ avaticaType.columnClassName());
+ }
+
+ private ColumnMetaData.AvaticaType avaticaType(JavaTypeFactory typeFactory,
+ RelDataType type, RelDataType fieldType) {
+ final String typeName = getTypeName(type);
+ if (type.getComponentType() != null) {
+ final ColumnMetaData.AvaticaType componentType =
+ avaticaType(typeFactory, type.getComponentType(), null);
+ final Type clazz = typeFactory.getJavaClass(type.getComponentType());
+ final ColumnMetaData.Rep rep = ColumnMetaData.Rep.of(clazz);
+ assert rep != null;
+ return ColumnMetaData.array(componentType, typeName, rep);
+ } else {
+ final int typeOrdinal = getTypeOrdinal(type);
+ switch (typeOrdinal) {
+ case Types.STRUCT:
+ final List<ColumnMetaData> columns = new ArrayList<>();
+ for (RelDataTypeField field : type.getFieldList()) {
+ columns.add(
+ metaData(typeFactory, field.getIndex(), field.getName(),
+ field.getType(), null, null));
+ }
+ return ColumnMetaData.struct(columns);
+ default:
+ final Type clazz =
+ typeFactory.getJavaClass(Util.first(fieldType, type));
+ final ColumnMetaData.Rep rep = ColumnMetaData.Rep.of(clazz);
+ assert rep != null;
+ return ColumnMetaData.scalar(typeOrdinal, typeName, rep);
+ }
+ }
+ }
+
+ private static String origin(List<String> origins, int offsetFromEnd) {
+ return origins == null || offsetFromEnd >= origins.size()
+ ? null
+ : origins.get(origins.size() - 1 - offsetFromEnd);
+ }
+
+ private int getTypeOrdinal(RelDataType type) {
+ return type.getSqlTypeName().getJdbcOrdinal();
+ }
+
+ private static String getClassName(RelDataType type) {
+ return null;
+ }
+
+ private static int getScale(RelDataType type) {
+ return type.getScale() == RelDataType.SCALE_NOT_SPECIFIED
+ ? 0
+ : type.getScale();
+ }
+
+ private static int getPrecision(RelDataType type) {
+ return type.getPrecision() == RelDataType.PRECISION_NOT_SPECIFIED
+ ? 0
+ : type.getPrecision();
+ }
+
+ /** Returns the type name in string form. Does not include precision, scale
+ * or whether nulls are allowed. Example: "DECIMAL" not "DECIMAL(7, 2)";
+ * "INTEGER" not "JavaType(int)". */
+ private static String getTypeName(RelDataType type) {
+ final SqlTypeName sqlTypeName = type.getSqlTypeName();
+ switch (sqlTypeName) {
+ case ARRAY:
+ case MULTISET:
+ case MAP:
+ case ROW:
+ return type.toString(); // e.g. "INTEGER ARRAY"
+ case INTERVAL_YEAR_MONTH:
+ return "INTERVAL_YEAR_TO_MONTH";
+ case INTERVAL_DAY_HOUR:
+ return "INTERVAL_DAY_TO_HOUR";
+ case INTERVAL_DAY_MINUTE:
+ return "INTERVAL_DAY_TO_MINUTE";
+ case INTERVAL_DAY_SECOND:
+ return "INTERVAL_DAY_TO_SECOND";
+ case INTERVAL_HOUR_MINUTE:
+ return "INTERVAL_HOUR_TO_MINUTE";
+ case INTERVAL_HOUR_SECOND:
+ return "INTERVAL_HOUR_TO_SECOND";
+ case INTERVAL_MINUTE_SECOND:
+ return "INTERVAL_MINUTE_TO_SECOND";
+ default:
+ return sqlTypeName.getName(); // e.g. "DECIMAL", "INTERVAL_YEAR_MONTH"
+ }
+ }
+
+ protected void populateMaterializations(Context context,
+ RelOptPlanner planner, Prepare.Materialization materialization) {
+ // REVIEW: initialize queryRel and tableRel inside MaterializationService,
+ // not here?
+ try {
+ final CalciteSchema schema = materialization.materializedTable.schema;
+ CalciteCatalogReader catalogReader =
+ new CalciteCatalogReader(
+ schema.root(),
+ context.config().caseSensitive(),
+ materialization.viewSchemaPath,
+ context.getTypeFactory());
+ final CalciteMaterializer materializer =
+ new CalciteMaterializer(this, context, catalogReader, schema, planner,
+ createConvertletTable());
+ materializer.populate(materialization);
+ } catch (Exception e) {
+ throw new RuntimeException("While populating materialization "
+ + materialization.materializedTable.path(), e);
+ }
+ }
+
+ private static RelDataType makeStruct(
+ RelDataTypeFactory typeFactory,
+ RelDataType type) {
+ if (type.isStruct()) {
+ return type;
+ }
+ return typeFactory.builder().add("$0", type).build();
+ }
+
+ /** Executes a prepare action. */
+ public <R> R perform(CalciteServerStatement statement,
+ Frameworks.PrepareAction<R> action) {
+ final CalcitePrepare.Context prepareContext =
+ statement.createPrepareContext();
+ final JavaTypeFactory typeFactory = prepareContext.getTypeFactory();
+ final CalciteSchema schema =
+ action.getConfig().getDefaultSchema() != null
+ ? CalciteSchema.from(action.getConfig().getDefaultSchema())
+ : prepareContext.getRootSchema();
+ CalciteCatalogReader catalogReader =
+ new CalciteCatalogReader(schema.root(),
+ prepareContext.config().caseSensitive(),
+ schema.path(null),
+ typeFactory);
+ final RexBuilder rexBuilder = new RexBuilder(typeFactory);
+ final RelOptPlanner planner =
+ createPlanner(prepareContext,
+ action.getConfig().getContext(),
+ action.getConfig().getCostFactory());
+ final RelOptCluster cluster = createCluster(planner, rexBuilder);
+ return action.apply(cluster, catalogReader,
+ prepareContext.getRootSchema().plus(), statement);
+ }
+
+ /** Holds state for the process of preparing a SQL statement. */
+ static class CalcitePreparingStmt extends Prepare
+ implements RelOptTable.ViewExpander {
+ protected final RelOptPlanner planner;
+ protected final RexBuilder rexBuilder;
+ protected final CalcitePrepareImpl prepare;
+ protected final CalciteSchema schema;
+ protected final RelDataTypeFactory typeFactory;
+ protected final SqlRexConvertletTable convertletTable;
+ private final EnumerableRel.Prefer prefer;
+ private final Map<String, Object> internalParameters =
+ Maps.newLinkedHashMap();
+ private int expansionDepth;
+ private SqlValidator sqlValidator;
+
+ public CalcitePreparingStmt(CalcitePrepareImpl prepare,
+ Context context,
+ CatalogReader catalogReader,
+ RelDataTypeFactory typeFactory,
+ CalciteSchema schema,
+ EnumerableRel.Prefer prefer,
+ RelOptPlanner planner,
+ Convention resultConvention,
+ SqlRexConvertletTable convertletTable) {
+ super(context, catalogReader, resultConvention);
+ this.prepare = prepare;
+ this.schema = schema;
+ this.prefer = prefer;
+ this.planner = planner;
+ this.typeFactory = typeFactory;
+ this.convertletTable = convertletTable;
+ this.rexBuilder = new RexBuilder(typeFactory);
+ }
+
+ @Override protected void init(Class runtimeContextClass) {
+ }
+
+ public PreparedResult prepareQueryable(
+ final Queryable queryable,
+ RelDataType resultType) {
+ return prepare_(
+ new Supplier<RelNode>() {
+ public RelNode get() {
+ final RelOptCluster cluster =
+ prepare.createCluster(planner, rexBuilder);
+ return new LixToRelTranslator(cluster, CalcitePreparingStmt.this)
+ .translate(queryable);
+ }
+ }, resultType);
+ }
+
+ public PreparedResult prepareRel(final RelNode rel) {
+ return prepare_(
+ new Supplier<RelNode>() {
+ public RelNode get() {
+ return rel;
+ }
+ }, rel.getRowType());
+ }
+
+ private PreparedResult prepare_(Supplier<RelNode> fn,
+ RelDataType resultType) {
+ queryString = null;
+ Class runtimeContextClass = Object.class;
+ init(runtimeContextClass);
+
+ final RelNode rel = fn.get();
+ final RelDataType rowType = rel.getRowType();
+ final List<Pair<Integer, String>> fields =
+ Pair.zip(ImmutableIntList.identity(rowType.getFieldCount()),
+ rowType.getFieldNames());
+ final RelCollation collation =
+ rel instanceof Sort
+ ? ((Sort) rel).collation
+ : RelCollations.EMPTY;
+ RelRoot root = new RelRoot(rel, resultType, SqlKind.SELECT, fields,
+ collation);
+
+ if (timingTracer != null) {
+ timingTracer.traceTime("end sql2rel");
+ }
+
+ final RelDataType jdbcType =
+ makeStruct(rexBuilder.getTypeFactory(), resultType);
+ fieldOrigins = Collections.nCopies(jdbcType.getFieldCount(), null);
+ parameterRowType = rexBuilder.getTypeFactory().builder().build();
+
+ // Structured type flattening, view expansion, and plugging in
+ // physical storage.
+ root = root.withRel(flattenTypes(root.rel, true));
+
+ // Trim unused fields.
+ root = trimUnusedFields(root);
+
+ final List<Materialization> materializations = ImmutableList.of();
+ final List<CalciteSchema.LatticeEntry> lattices = ImmutableList.of();
+ root = optimize(root, materializations, lattices);
+
+ if (timingTracer != null) {
+ timingTracer.traceTime("end optimization");
+ }
+
+ return implement(root);
+ }
+
+ @Override protected SqlToRelConverter getSqlToRelConverter(
+ SqlValidator validator,
+ CatalogReader catalogReader,
+ SqlToRelConverter.Config config) {
+ final RelOptCluster cluster = prepare.createCluster(planner, rexBuilder);
+ SqlToRelConverter sqlToRelConverter =
+ new SqlToRelConverter(this, validator, catalogReader, cluster,
+ convertletTable, config);
+ return sqlToRelConverter;
+ }
+
+ @Override public RelNode flattenTypes(
+ RelNode rootRel,
+ boolean restructure) {
+ final SparkHandler spark = context.spark();
+ if (spark.enabled()) {
+ return spark.flattenTypes(planner, rootRel, restructure);
+ }
+ return rootRel;
+ }
+
+ @Override protected RelNode decorrelate(SqlToRelConverter sqlToRelConverter,
+ SqlNode query, RelNode rootRel) {
+ return sqlToRelConverter.decorrelate(query, rootRel);
+ }
+
+ @Override public RelRoot expandView(RelDataType rowType, String queryString,
+ List<String> schemaPath, List<String> viewPath) {
+ expansionDepth++;
+
+ SqlParser parser = prepare.createParser(queryString);
+ SqlNode sqlNode;
+ try {
+ sqlNode = parser.parseQuery();
+ } catch (SqlParseException e) {
+ throw new RuntimeException("parse failed", e);
+ }
+ // View may have different schema path than current connection.
+ final CatalogReader catalogReader =
+ this.catalogReader.withSchemaPath(schemaPath);
+ SqlValidator validator = createSqlValidator(catalogReader);
+ SqlNode sqlNode1 = validator.validate(sqlNode);
+ final SqlToRelConverter.Config config = SqlToRelConverter.configBuilder()
+ .withTrimUnusedFields(true).build();
+ SqlToRelConverter sqlToRelConverter =
+ getSqlToRelConverter(validator, catalogReader, config);
+ RelRoot root =
+ sqlToRelConverter.convertQuery(sqlNode1, true, false);
+
+ --expansionDepth;
+ return root;
+ }
+
+ protected SqlValidator createSqlValidator(CatalogReader catalogReader) {
+ return prepare.createSqlValidator(context,
+ (CalciteCatalogReader) catalogReader);
+ }
+
+ @Override protected SqlValidator getSqlValidator() {
+ if (sqlValidator == null) {
+ sqlValidator = createSqlValidator(catalogReader);
+ }
+ return sqlValidator;
+ }
+
+ @Override protected PreparedResult createPreparedExplanation(
+ RelDataType resultType,
+ RelDataType parameterRowType,
+ RelRoot root,
+ SqlExplainFormat format,
+ SqlExplainLevel detailLevel) {
+ return new CalcitePreparedExplain(resultType, parameterRowType, root,
+ format, detailLevel);
+ }
+
+ @Override protected PreparedResult implement(RelRoot root) {
+ RelDataType resultType = root.rel.getRowType();
+ boolean isDml = root.kind.belongsTo(SqlKind.DML);
+ final Bindable bindable;
+ if (resultConvention == BindableConvention.INSTANCE) {
+ bindable = Interpreters.bindable(root.rel);
+ } else {
+ EnumerableRel enumerable = (EnumerableRel) root.rel;
+ if (!root.isRefTrivial()) {
+ final List<RexNode> projects = new ArrayList<>();
+ final RexBuilder rexBuilder = enumerable.getCluster().getRexBuilder();
+ for (int field : Pair.left(root.fields)) {
+ projects.add(rexBuilder.makeInputRef(enumerable, field));
+ }
+ RexProgram program = RexProgram.create(enumerable.getRowType(),
+ projects, null, root.validatedRowType, rexBuilder);
+ enumerable = EnumerableCalc.create(enumerable, program);
+ }
+
+ try {
+ CatalogReader.THREAD_LOCAL.set(catalogReader);
+ bindable = EnumerableInterpretable.toBindable(internalParameters,
+ context.spark(), enumerable, prefer);
+ } finally {
+ CatalogReader.THREAD_LOCAL.remove();
+ }
+ }
+
+ if (timingTracer != null) {
+ timingTracer.traceTime("end codegen");
+ }
+
+ if (timingTracer != null) {
+ timingTracer.traceTime("end compilation");
+ }
+
+ return new PreparedResultImpl(
+ resultType,
+ parameterRowType,
+ fieldOrigins,
+ root.collation.getFieldCollations().isEmpty()
+ ? ImmutableList.<RelCollation>of()
+ : ImmutableList.of(root.collation),
+ root.rel,
+ mapTableModOp(isDml, root.kind),
+ isDml) {
+ public String getCode() {
+ throw new UnsupportedOperationException();
+ }
+
+ public Bindable getBindable(Meta.CursorFactory cursorFactory) {
+ return bindable;
+ }
+
+ public Type getElementType() {
+ return ((Typed) bindable).getElementType();
+ }
+ };
+ }
+
+ @Override protected List<Materialization> getMaterializations() {
+ final List<Prepare.Materialization> materializations =
+ context.config().materializationsEnabled()
+ ? MaterializationService.instance().query(schema)
+ : ImmutableList.<Prepare.Materialization>of();
+ for (Prepare.Materialization materialization : materializations) {
+ prepare.populateMaterializations(context, planner, materialization);
+ }
+ return materializations;
+ }
+
+ @Override protected List<LatticeEntry> getLattices() {
+ return Schemas.getLatticeEntries(schema);
+ }
+ }
+
+ /** An {@code EXPLAIN} statement, prepared and ready to execute. */
+ private static class CalcitePreparedExplain extends Prepare.PreparedExplain {
+ public CalcitePreparedExplain(
+ RelDataType resultType,
+ RelDataType parameterRowType,
+ RelRoot root,
+ SqlExplainFormat format,
+ SqlExplainLevel detailLevel) {
+ super(resultType, parameterRowType, root, format, detailLevel);
+ }
+
+ public Bindable getBindable(final Meta.CursorFactory cursorFactory) {
+ final String explanation = getCode();
+ return new Bindable() {
+ public Enumerable bind(DataContext dataContext) {
+ switch (cursorFactory.style) {
+ case ARRAY:
+ return Linq4j.singletonEnumerable(new String[] {explanation});
+ case OBJECT:
+ default:
+ return Linq4j.singletonEnumerable(explanation);
+ }
+ }
+ };
+ }
+ }
+
+ /** Translator from Java AST to {@link RexNode}. */
+ interface ScalarTranslator {
+ RexNode toRex(BlockStatement statement);
+ List<RexNode> toRexList(BlockStatement statement);
+ RexNode toRex(Expression expression);
+ ScalarTranslator bind(List<ParameterExpression> parameterList,
+ List<RexNode> values);
+ }
+
+ /** Basic translator. */
+ static class EmptyScalarTranslator implements ScalarTranslator {
+ private final RexBuilder rexBuilder;
+
+ public EmptyScalarTranslator(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ }
+
+ public static ScalarTranslator empty(RexBuilder builder) {
+ return new EmptyScalarTranslator(builder);
+ }
+
+ public List<RexNode> toRexList(BlockStatement statement) {
+ final List<Expression> simpleList = simpleList(statement);
+ final List<RexNode> list = new ArrayList<>();
+ for (Expression expression1 : simpleList) {
+ list.add(toRex(expression1));
+ }
+ return list;
+ }
+
+ public RexNode toRex(BlockStatement statement) {
+ return toRex(Blocks.simple(statement));
+ }
+
+ private static List<Expression> simpleList(BlockStatement statement) {
+ Expression simple = Blocks.simple(statement);
+ if (simple instanceof NewExpression) {
+ NewExpression newExpression = (NewExpression) simple;
+ return newExpression.arguments;
+ } else {
+ return Collections.singletonList(simple);
+ }
+ }
+
+ public RexNode toRex(Expression expression) {
+ switch (expression.getNodeType()) {
+ case MemberAccess:
+ // Case-sensitive name match because name was previously resolved.
+ return rexBuilder.makeFieldAccess(
+ toRex(
+ ((MemberExpression) expression).expression),
+ ((MemberExpression) expression).field.getName(),
+ true);
+ case GreaterThan:
+ return binary(expression, SqlStdOperatorTable.GREATER_THAN);
+ case LessThan:
+ return binary(expression, SqlStdOperatorTable.LESS_THAN);
+ case Parameter:
+ return parameter((ParameterExpression) expression);
+ case Call:
+ MethodCallExpression call = (MethodCallExpression) expression;
+ SqlOperator operator =
+ RexToLixTranslator.JAVA_TO_SQL_METHOD_MAP.get(call.method);
+ if (operator != null) {
+ return rexBuilder.makeCall(
+ type(call),
+ operator,
+ toRex(
+ Expressions.<Expression>list()
+ .appendIfNotNull(call.targetExpression)
+ .appendAll(call.expressions)));
+ }
+ throw new RuntimeException(
+ "Could translate call to method " + call.method);
+ case Constant:
+ final ConstantExpression constant =
+ (ConstantExpression) expression;
+ Object value = constant.value;
+ if (value instanceof Number) {
+ Number number = (Number) value;
+ if (value instanceof Double || value instanceof Float) {
+ return rexBuilder.makeApproxLiteral(
+ BigDecimal.valueOf(number.doubleValue()));
+ } else if (value instanceof BigDecimal) {
+ return rexBuilder.makeExactLiteral((BigDecimal) value);
+ } else {
+ return rexBuilder.makeExactLiteral(
+ BigDecimal.valueOf(number.longValue()));
+ }
+ } else if (value instanceof Boolean) {
+ return rexBuilder.makeLiteral((Boolean) value);
+ } else {
+ return rexBuilder.makeLiteral(constant.toString());
+ }
+ default:
+ throw new UnsupportedOperationException(
+ "unknown expression type " + expression.getNodeType() + " "
+ + expression);
+ }
+ }
+
+ private RexNode binary(Expression expression, SqlBinaryOperator op) {
+ BinaryExpression call = (BinaryExpression) expression;
+ return rexBuilder.makeCall(type(call), op,
+ toRex(ImmutableList.of(call.expression0, call.expression1)));
+ }
+
+ private List<RexNode> toRex(List<Expression> expressions) {
+ final List<RexNode> list = new ArrayList<>();
+ for (Expression expression : expressions) {
+ list.add(toRex(expression));
+ }
+ return list;
+ }
+
+ protected RelDataType type(Expression expression) {
+ final Type type = expression.getType();
+ return ((JavaTypeFactory) rexBuilder.getTypeFactory()).createType(type);
+ }
+
+ public ScalarTranslator bind(
+ List<ParameterExpression> parameterList, List<RexNode> values) {
+ return new LambdaScalarTranslator(
+ rexBuilder, parameterList, values);
+ }
+
+ public RexNode parameter(ParameterExpression param) {
+ throw new RuntimeException("unknown parameter " + param);
+ }
+ }
+
+ /** Translator that looks for parameters. */
+ private static class LambdaScalarTranslator extends EmptyScalarTranslator {
+ private final List<ParameterExpression> parameterList;
+ private final List<RexNode> values;
+
+ public LambdaScalarTranslator(
+ RexBuilder rexBuilder,
+ List<ParameterExpression> parameterList,
+ List<RexNode> values) {
+ super(rexBuilder);
+ this.parameterList = parameterList;
+ this.values = values;
+ }
+
+ public RexNode parameter(ParameterExpression param) {
+ int i = parameterList.indexOf(param);
+ if (i >= 0) {
+ return values.get(i);
+ }
+ throw new RuntimeException("unknown parameter " + param);
+ }
+ }
+}
+
+// End CalcitePrepareImpl.java
http://git-wip-us.apache.org/repos/asf/kylin/blob/56e9fb9e/atopcalcite/src/main/java/org/apache/calcite/prepare/OnlyPrepareEarlyAbortException.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/prepare/OnlyPrepareEarlyAbortException.java b/atopcalcite/src/main/java/org/apache/calcite/prepare/OnlyPrepareEarlyAbortException.java
new file mode 100644
index 0000000..8493484
--- /dev/null
+++ b/atopcalcite/src/main/java/org/apache/calcite/prepare/OnlyPrepareEarlyAbortException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.calcite.prepare;
+
+import org.apache.calcite.jdbc.CalcitePrepare;
+
+public class OnlyPrepareEarlyAbortException extends RuntimeException {
+
+ private CalcitePrepare.Context context;
+ private org.apache.calcite.jdbc.CalcitePrepare.ParseResult preparedResult;
+
+ public OnlyPrepareEarlyAbortException(CalcitePrepare.Context context,
+ org.apache.calcite.jdbc.CalcitePrepare.ParseResult preparedResult) {
+ this.context = context;
+ this.preparedResult = preparedResult;
+ }
+
+ public CalcitePrepare.Context getContext() {
+ return context;
+ }
+
+ public CalcitePrepare.ParseResult getPreparedResult() {
+ return preparedResult;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/56e9fb9e/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 0da92c7..35dcee4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -22,10 +22,13 @@ import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import javax.servlet.http.HttpServletResponse;
+import com.google.common.collect.Maps;
import org.apache.commons.io.IOUtils;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.querymeta.TableMeta;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -76,6 +79,10 @@ public class QueryController extends BasicController {
@RequestMapping(value = "/query/prestate", method = RequestMethod.POST, produces = "application/json")
@ResponseBody
public SQLResponse prepareQuery(@RequestBody PrepareSqlRequest sqlRequest) {
+ Map<String, String> toggles = Maps.newHashMap();
+ toggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
+ BackdoorToggles.addToggles(toggles);
+
return queryService.doQueryWithCache(sqlRequest);
}
@@ -106,7 +113,7 @@ public class QueryController extends BasicController {
@ResponseBody
public void downloadQueryResult(@PathVariable String format, SQLRequest sqlRequest, HttpServletResponse response) {
SQLResponse result = queryService.doQueryWithCache(sqlRequest);
- response.setContentType("text/" + format + ";charset=ansi");
+ response.setContentType("text/" + format + ";charset=utf-8");
response.setHeader("Content-Disposition", "attachment; filename=\"result." + format + "\"");
ICsvListWriter csvWriter = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/56e9fb9e/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index b0ed171..1b118a3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -47,6 +47,12 @@ import javax.annotation.PostConstruct;
import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
+import org.apache.calcite.config.CalciteConnectionConfig;
+import org.apache.calcite.jdbc.CalcitePrepare;
+import org.apache.calcite.prepare.CalcitePrepareImpl;
+import org.apache.calcite.prepare.OnlyPrepareEarlyAbortException;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.sql.type.BasicSqlType;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.kylin.common.KylinConfig;
@@ -117,12 +123,10 @@ import net.sf.ehcache.Element;
@Component("queryService")
public class QueryService extends BasicService {
- private static final Logger logger = LoggerFactory.getLogger(QueryService.class);
-
public static final String SUCCESS_QUERY_CACHE = "StorageCache";
public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache";
public static final String QUERY_STORE_PATH_PREFIX = "/query/";
-
+ private static final Logger logger = LoggerFactory.getLogger(QueryService.class);
final BadQueryDetector badQueryDetector = new BadQueryDetector();
final ResourceStore queryStore;
@@ -140,16 +144,27 @@ public class QueryService extends BasicService {
@Autowired
private AclUtil aclUtil;
- @PostConstruct
- public void init() throws IOException {
- Preconditions.checkNotNull(cacheManager, "cacheManager is not injected yet");
- }
-
public QueryService() {
queryStore = ResourceStore.getStore(getConfig());
badQueryDetector.start();
}
+ protected static void close(ResultSet resultSet, Statement stat, Connection conn) {
+ OLAPContext.clearParameter();
+ DBUtils.closeQuietly(resultSet);
+ DBUtils.closeQuietly(stat);
+ DBUtils.closeQuietly(conn);
+ }
+
+ private static String getQueryKeyById(String creator) {
+ return QUERY_STORE_PATH_PREFIX + creator;
+ }
+
+ @PostConstruct
+ public void init() throws IOException {
+ Preconditions.checkNotNull(cacheManager, "cacheManager is not injected yet");
+ }
+
public List<TableMeta> getMetadata(String project) throws SQLException {
return getMetadata(getCubeManager(), project, true);
}
@@ -535,6 +550,7 @@ public class QueryService extends BasicService {
return getMetadataV2(getCubeManager(), project, true);
}
+ @SuppressWarnings("checkstyle:methodlength")
protected List<TableMetaWithType> getMetadataV2(CubeManager cubeMgr, String project, boolean cubedOnly) throws SQLException, IOException {
//Message msg = MsgPicker.getMsg();
@@ -695,21 +711,15 @@ public class QueryService extends BasicService {
try {
conn = cacheService.getOLAPDataSource(sqlRequest.getProject()).getConnection();
- if (sqlRequest instanceof PrepareSqlRequest) {
- PreparedStatement preparedState = conn.prepareStatement(correctedSql);
- processStatementAttr(preparedState, sqlRequest);
-
- for (int i = 0; i < ((PrepareSqlRequest) sqlRequest).getParams().length; i++) {
- setParam(preparedState, i + 1, ((PrepareSqlRequest) sqlRequest).getParams()[i]);
- }
-
- resultSet = preparedState.executeQuery();
- } else {
- stat = conn.createStatement();
- processStatementAttr(stat, sqlRequest);
- resultSet = stat.executeQuery(correctedSql);
+ // special case for prepare query.
+ if (BackdoorToggles.getPrepareOnly()) {
+ return getPrepareOnlySqlResponse(correctedSql, conn, isAdHoc, results, columnMetas);
}
+ stat = conn.createStatement();
+ processStatementAttr(stat, sqlRequest);
+ resultSet = stat.executeQuery(correctedSql);
+
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();
@@ -733,6 +743,50 @@ public class QueryService extends BasicService {
close(resultSet, stat, conn);
}
+ return getSqlResponse(isAdHoc, results, columnMetas);
+ }
+
+ private SQLResponse getPrepareOnlySqlResponse(String correctedSql, Connection conn,
+ Boolean isAdHoc, List<List<String>> results, List<SelectedColumnMeta> columnMetas)
+ throws SQLException {
+
+ CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(true);
+
+ try {
+ conn.prepareStatement(correctedSql);
+ throw new IllegalStateException("Should have thrown OnlyPrepareEarlyAbortException");
+ } catch (Exception e) {
+ Throwable rootCause = ExceptionUtils.getRootCause(e);
+ if (rootCause != null && rootCause instanceof OnlyPrepareEarlyAbortException) {
+ OnlyPrepareEarlyAbortException abortException = (OnlyPrepareEarlyAbortException) rootCause;
+ CalcitePrepare.Context context = abortException.getContext();
+ CalcitePrepare.ParseResult preparedResult = abortException.getPreparedResult();
+ List<RelDataTypeField> fieldList = preparedResult.rowType.getFieldList();
+
+ CalciteConnectionConfig config = context.config();
+
+ // Fill in selected column meta
+ for (int i = 0; i < fieldList.size(); ++i) {
+
+ RelDataTypeField field = fieldList.get(i);
+ String columnName = field.getKey();
+ BasicSqlType basicSqlType = (BasicSqlType) field.getValue();
+
+ columnMetas.add(new SelectedColumnMeta(false, config.caseSensitive(), false, false, basicSqlType.isNullable() ? 1 : 0, true, basicSqlType.getPrecision(), columnName, columnName, null, null, null, basicSqlType.getPrecision(), basicSqlType.getScale(), basicSqlType.getSqlTypeName().getJdbcOrdinal(), basicSqlType.getSqlTypeName().getName(), true, false, false));
+ }
+
+ } else {
+ throw e;
+ }
+ } finally {
+ CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(false);
+ }
+
+ return getSqlResponse(isAdHoc, results, columnMetas);
+ }
+
+ private SQLResponse getSqlResponse(Boolean isAdHoc, List<List<String>> results, List<SelectedColumnMeta> columnMetas) {
+
boolean isPartialResult = false;
StringBuilder cubeSb = new StringBuilder();
StringBuilder logSb = new StringBuilder("Processed rows for each storageContext: ");
@@ -753,7 +807,6 @@ public class QueryService extends BasicService {
SQLResponse response = new SQLResponse(columnMetas, results, cubeSb.toString(), 0, false, null, isPartialResult, isAdHoc);
response.setTotalScanCount(QueryContext.current().getScannedRows());
response.setTotalScanBytes(QueryContext.current().getScannedBytes());
-
return response;
}
@@ -839,21 +892,10 @@ public class QueryService extends BasicService {
}
}
- protected static void close(ResultSet resultSet, Statement stat, Connection conn) {
- OLAPContext.clearParameter();
- DBUtils.closeQuietly(resultSet);
- DBUtils.closeQuietly(stat);
- DBUtils.closeQuietly(conn);
- }
-
public void setCacheManager(CacheManager cacheManager) {
this.cacheManager = cacheManager;
}
- private static String getQueryKeyById(String creator) {
- return QUERY_STORE_PATH_PREFIX + creator;
- }
-
private static class QueryRecordSerializer implements Serializer<QueryRecord> {
private static final QueryRecordSerializer serializer = new QueryRecordSerializer();