You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/09/18 05:44:10 UTC
[17/29] kylin git commit: minor, refine query error message
minor, refine query error message
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b97e16ad
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b97e16ad
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b97e16ad
Branch: refs/heads/security_update
Commit: b97e16ad8ad561e2f1ca8fe60fe7ad8c577401d8
Parents: 6d47976
Author: liveandevil <13...@qq.com>
Authored: Sat Sep 16 14:51:48 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Sat Sep 16 15:30:30 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 8 +-
.../kylin/cube/CubeCapabilityChecker.java | 54 +++--
.../org/apache/kylin/cube/CubeInstance.java | 10 -
.../kylin/cube/model/AggregationGroup.java | 7 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 13 +-
.../kylin/metadata/model/DataModelDesc.java | 6 +-
.../kylin/metadata/model/JoinTableDesc.java | 50 +++-
.../apache/kylin/metadata/model/JoinsTree.java | 34 ++-
.../metadata/realization/CapabilityResult.java | 75 +++++-
.../kylin/storage/hybrid/HybridInstance.java | 4 +-
.../apache/kylin/query/relnode/OLAPContext.java | 7 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 14 +-
.../apache/kylin/query/routing/QueryRouter.java | 25 +-
.../kylin/query/routing/RealizationCheck.java | 242 +++++++++++++++++++
.../kylin/query/routing/RealizationChooser.java | 42 +++-
.../rules/RemoveUncapableRealizationsRule.java | 5 +-
.../org/apache/kylin/query/util/QueryUtil.java | 7 +-
.../apache/kylin/rest/service/QueryService.java | 6 +-
.../rest/controller/QueryControllerTest.java | 7 +-
19 files changed, 534 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 990736f..ab19d2a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -290,7 +290,7 @@ abstract public class KylinConfigBase implements Serializable {
"org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory");
return (DistributedLockFactory) ClassUtil.newInstance(clsName);
}
-
+
public String getHBaseMappingAdapter() {
return getOptional("kylin.metadata.hbasemapping-adapter");
}
@@ -346,7 +346,7 @@ abstract public class KylinConfigBase implements Serializable {
public String getSegmentAdvisor() {
return getOptional("kylin.cube.segment-advisor", "org.apache.kylin.cube.CubeSegmentAdvisor");
}
-
+
public double getJobCuboidSizeRatio() {
return Double.parseDouble(getOptional("kylin.cube.size-estimate-ratio", "0.25"));
}
@@ -388,6 +388,10 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.cube.aggrgroup.is-mandatory-only-valid", "false"));
}
+ public int getCubeRowkeyMaxSize() {
+ return Integer.parseInt(getOptional("kylin.cube.rowkey.max-size", "63"));
+ }
+
public int getMaxBuildingSegments() {
return Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 47f268a..5dffd96 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -70,7 +70,8 @@ public class CubeCapabilityChecker {
//1. dimension as measure
if (!unmatchedAggregations.isEmpty()) {
- tryDimensionAsMeasures(unmatchedAggregations, result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
+ tryDimensionAsMeasures(unmatchedAggregations, result,
+ cube.getDescriptor().listDimensionColumnsIncludingDerived());
}
} else {
//for non query-on-facttable
@@ -103,36 +104,47 @@ public class CubeCapabilityChecker {
if (!unmatchedDimensions.isEmpty()) {
logger.info("Exclude cube " + cube.getName() + " because unmatched dimensions: " + unmatchedDimensions);
+ result.incapableCause = CapabilityResult.IncapableCause.unmatchedDimensions(unmatchedDimensions);
return result;
}
if (!unmatchedAggregations.isEmpty()) {
logger.info("Exclude cube " + cube.getName() + " because unmatched aggregations: " + unmatchedAggregations);
+ result.incapableCause = CapabilityResult.IncapableCause.unmatchedAggregations(unmatchedAggregations);
return result;
}
- if (cube.getStorageType() == IStorageAware.ID_HBASE && MassInTupleFilter.containsMassInTupleFilter(digest.filter)) {
- logger.info("Exclude cube " + cube.getName() + " because only v2 storage + v2 query engine supports massin");
+ if (cube.getStorageType() == IStorageAware.ID_HBASE
+ && MassInTupleFilter.containsMassInTupleFilter(digest.filter)) {
+ logger.info(
+ "Exclude cube " + cube.getName() + " because only v2 storage + v2 query engine supports massin");
+ result.incapableCause = CapabilityResult.IncapableCause.create(CapabilityResult.IncapableType.UNSUPPORT_MASSIN);
return result;
}
if (digest.limitPrecedesAggr) {
logger.info("Exclude cube " + cube.getName() + " because there's limit preceding aggregation");
+ result.incapableCause = CapabilityResult.IncapableCause.create(CapabilityResult.IncapableType.LIMIT_PRECEDE_AGGR);
return result;
}
if (digest.isRawQuery && rootFactTable.equals(digest.factTable)) {
- result.influences.add(new CapabilityInfluence() {
- @Override
- public double suggestCostMultiplier() {
- return 100;
- }
+ if (cube.getConfig().isDisableCubeNoAggSQL()) {
+ result.incapableCause = CapabilityResult.IncapableCause.create(CapabilityResult.IncapableType.UNSUPPORT_RAWQUERY);
+ return result;
+ } else {
+ result.influences.add(new CapabilityInfluence() {
+ @Override
+ public double suggestCostMultiplier() {
+ return 100;
+ }
- @Override
- public MeasureDesc getInvolvedMeasure() {
- return null;
- }
- });
+ @Override
+ public MeasureDesc getInvolvedMeasure() {
+ return null;
+ }
+ });
+ }
}
// cost will be minded by caller
@@ -164,7 +176,8 @@ public class CubeCapabilityChecker {
return result;
}
- private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, CapabilityResult result, Set<TblColRef> dimCols) {
+ private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, CapabilityResult result,
+ Set<TblColRef> dimCols) {
Iterator<FunctionDesc> it = unmatchedAggregations.iterator();
while (it.hasNext()) {
@@ -182,7 +195,8 @@ public class CubeCapabilityChecker {
continue;
}
List<TblColRef> neededCols = parameterDesc.getColRefs();
- if (neededCols.size() > 0 && dimCols.containsAll(neededCols) && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) {
+ if (neededCols.size() > 0 && dimCols.containsAll(neededCols)
+ && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) {
result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc));
it.remove();
continue;
@@ -191,7 +205,9 @@ public class CubeCapabilityChecker {
}
// custom measure types can cover unmatched dimensions or measures
- private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result) {
+ private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions,
+ Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube,
+ CapabilityResult result) {
CubeDesc cubeDesc = cube.getDescriptor();
List<String> influencingMeasures = Lists.newArrayList();
for (MeasureDesc measure : cubeDesc.getMeasures()) {
@@ -202,14 +218,16 @@ public class CubeCapabilityChecker {
if (measureType instanceof BasicMeasureType)
continue;
- CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations, digest, measure);
+ CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations,
+ digest, measure);
if (inf != null) {
result.influences.add(inf);
influencingMeasures.add(measure.getName() + "@" + measureType.getClass());
}
}
if (influencingMeasures.size() != 0)
- logger.info("Cube {} CapabilityInfluences: {}", cube.getCanonicalName(), StringUtils.join(influencingMeasures, ","));
+ logger.info("Cube {} CapabilityInfluences: {}", cube.getCanonicalName(),
+ StringUtils.join(influencingMeasures, ","));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 7d539c6..ac41970 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -404,7 +404,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
@Override
public CapabilityResult isCapable(SQLDigest digest) {
CapabilityResult result = CubeCapabilityChecker.check(this, digest);
- result = localCapacityCheck(digest, result);
if (result.capable) {
result.cost = getCost(digest);
for (CapabilityInfluence i : result.influences) {
@@ -416,15 +415,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return result;
}
- private CapabilityResult localCapacityCheck(SQLDigest digest, CapabilityResult originResult) {
- if (this.getDescriptor().getConfig().isDisableCubeNoAggSQL()) {
- CapabilityResult notCap = new CapabilityResult();
- notCap.capable = false;
- return digest.aggregations.isEmpty() ? notCap : originResult ;
- }
- return originResult;
- }
-
public int getCost(SQLDigest digest) {
int calculatedCost = cost;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 8c90b8e..d473858 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -122,7 +122,8 @@ public class AggregationGroup implements Serializable {
// check no dup
Set<String> set = new HashSet<>(Arrays.asList(names));
if (set.size() < names.length)
- throw new IllegalStateException("Columns in aggrgroup must not contain duplication: " + Arrays.asList(names));
+ throw new IllegalStateException(
+ "Columns in aggrgroup must not contain duplication: " + Arrays.asList(names));
}
private void buildPartialCubeFullMask(RowKeyDesc rowKeyDesc) {
@@ -325,12 +326,16 @@ public class AggregationGroup implements Serializable {
normalDims.removeAll(jointDims);
combination = combination * (1L << normalDims.size());
+
if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid() && !mandatoryDims.isEmpty()) {
combination += 1;
}
combination -= 1; // not include cuboid 0
}
+ if (combination < 0) { // overflow
+ combination = Long.MAX_VALUE - 1;
+ }
return combination;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 46683c4..f2dab51 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -96,8 +96,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
}
}
- public static final int MAX_ROWKEY_SIZE = 64;
-
public enum DeriveType implements java.io.Serializable {
LOOKUP, PK_FK, EXTENDED_COLUMN
}
@@ -191,7 +189,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap();
private Map<TblColRef, DeriveInfo> extendedColumnToHosts = Maps.newHashMap();
-
+
transient private CuboidScheduler cuboidScheduler = null;
public boolean isEnableSharding() {
@@ -563,9 +561,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
checkArgument(StringUtils.isNotBlank(name), "CubeDesc name is blank");
checkArgument(StringUtils.isNotBlank(modelName), "CubeDesc (%s) has blank model name", name);
- checkArgument(this.rowkey.getRowKeyColumns().length < MAX_ROWKEY_SIZE,
- "Too many rowkeys (%s) in CubeDesc, please try to reduce dimension number or adopt derived dimensions",
- this.rowkey.getRowKeyColumns().length);
// note CubeDesc.name == CubeInstance.name
List<ProjectInstance> ownerPrj = ProjectManager.getInstance(config).findProjects(RealizationType.CUBE, name);
@@ -582,6 +577,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
this.config = KylinConfigExt.createInstance(config, overrideKylinProps);
+ checkArgument(this.rowkey.getRowKeyColumns().length <= this.config.getCubeRowkeyMaxSize(),
+ "Too many rowkeys (%s) in CubeDesc, please try to reduce dimension number or adopt derived dimensions",
+ this.rowkey.getRowKeyColumns().length);
+
this.model = MetadataManager.getInstance(config).getDataModelDesc(modelName);
checkNotNull(this.model, "DateModelDesc(%s) not found", modelName);
@@ -618,7 +617,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
hbaseMapping.init(this);
initMeasureReferenceToColumnFamily();
}
- }
+ }
// check all dimension columns are presented on rowkey
List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index b27d644..1897c0f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -843,12 +843,10 @@ public class DataModelDesc extends RootPersistentEntity {
return metrics;
}
- @Deprecated
public void setDimensions(List<ModelDimensionDesc> dimensions) {
this.dimensions = dimensions;
}
- @Deprecated
public void setMetrics(String[] metrics) {
this.metrics = metrics;
}
@@ -873,9 +871,11 @@ public class DataModelDesc extends RootPersistentEntity {
copy.dimensions = orig.dimensions;
copy.metrics = orig.metrics;
copy.filterCondition = orig.filterCondition;
- copy.partitionDesc = PartitionDesc.getCopyOf(orig.getPartitionDesc());
copy.capacity = orig.capacity;
copy.computedColumnDescs = orig.computedColumnDescs;
+ if (orig.getPartitionDesc() != null) {
+ copy.partitionDesc = PartitionDesc.getCopyOf(orig.getPartitionDesc());
+ }
copy.updateRandomUuid();
return copy;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
index 56c90bd..dc4710e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
@@ -37,36 +37,44 @@ public class JoinTableDesc implements Serializable {
@JsonProperty("kind")
@JsonInclude(JsonInclude.Include.NON_NULL)
private TableKind kind = TableKind.LOOKUP;
-
+
@JsonProperty("alias")
@JsonInclude(JsonInclude.Include.NON_NULL)
private String alias;
-
+
@JsonProperty("join")
private JoinDesc join;
-
+
private TableRef tableRef;
public String getTable() {
return table;
}
- void setTable(String table) {
+ public void setTable(String table) {
this.table = table;
}
public TableKind getKind() {
return kind;
}
-
- void setAlias(String alias) {
+
+ public void setKind(TableKind kind) {
+ this.kind = kind;
+ }
+
+ public void setAlias(String alias) {
this.alias = alias;
}
-
+
public String getAlias() {
return alias;
}
-
+
+ public void setJoin(JoinDesc join) {
+ this.join = join;
+ }
+
public JoinDesc getJoin() {
return join;
}
@@ -79,4 +87,30 @@ public class JoinTableDesc implements Serializable {
this.tableRef = ref;
}
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ JoinTableDesc that = (JoinTableDesc) o;
+
+ if (table != null ? !table.equals(that.table) : that.table != null)
+ return false;
+ if (kind != that.kind)
+ return false;
+ if (alias != null ? !alias.equals(that.alias) : that.alias != null)
+ return false;
+ return join != null ? join.equals(that.join) : that.join == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = table != null ? table.hashCode() : 0;
+ result = 31 * result + (kind != null ? kind.hashCode() : 0);
+ result = 31 * result + (alias != null ? alias.hashCode() : 0);
+ result = 31 * result + (join != null ? join.hashCode() : 0);
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index 4e7e8b8..a37c6be 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
public class JoinsTree implements Serializable {
private static final long serialVersionUID = 1L;
@@ -73,7 +74,8 @@ public class JoinsTree implements Serializable {
return matchUp.size();
}
- private boolean matchInTree(Chain chain, JoinsTree another, Map<String, String> constraints, Map<String, String> matchUp) {
+ private boolean matchInTree(Chain chain, JoinsTree another, Map<String, String> constraints,
+ Map<String, String> matchUp) {
String thisAlias = chain.table.getAlias();
if (matchUp.containsKey(thisAlias))
return true;
@@ -103,7 +105,8 @@ public class JoinsTree implements Serializable {
boolean matches = false;
if (chain.join == null) {
- matches = anotherChain.join == null && chain.table.getTableDesc().getIdentity().equals(anotherChain.table.getTableDesc().getIdentity());
+ matches = anotherChain.join == null
+ && chain.table.getTableDesc().getIdentity().equals(anotherChain.table.getTableDesc().getIdentity());
} else {
matches = chain.join.matches(anotherChain.join) && matchChain(chain.fkSide, anotherChain.fkSide, matchUp);
}
@@ -122,6 +125,21 @@ public class JoinsTree implements Serializable {
return chain.join;
}
+ public List<Chain> unmatchedChain(JoinsTree another, Map<String, String> constraints) {
+ Map<String, String> matchUp = new HashMap<>();
+ List<Chain> unmatchedChainList = Lists.newArrayList();
+ for (Chain chain : tableChains.values()) {
+ if (matchInTree(chain, another, constraints, matchUp) == false)
+ unmatchedChainList.add(chain);
+ }
+
+ return unmatchedChainList;
+ }
+
+ public Map<String, Chain> getTableChains() {
+ return tableChains;
+ }
+
public static class Chain implements Serializable {
private static final long serialVersionUID = 1L;
@@ -138,6 +156,18 @@ public class JoinsTree implements Serializable {
Preconditions.checkArgument(fkSide.table == join.getFKSide());
}
}
+
+ public JoinDesc getJoin() {
+ return join;
+ }
+
+ public TableRef getTable() {
+ return table;
+ }
+
+ public Chain getFkSide() {
+ return fkSide;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java
index a2bece8..ba21939 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java
@@ -18,29 +18,42 @@
package org.apache.kylin.metadata.realization;
+import java.util.Collection;
import java.util.List;
import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.collect.Lists;
-import org.apache.kylin.metadata.model.MeasureDesc;
public class CapabilityResult {
- /** Is capable or not */
+ /**
+ * Is capable or not
+ */
public boolean capable;
- /** The smaller the cost, the more capable the realization */
+ /**
+ * The smaller the cost, the more capable the realization
+ */
public int cost;
/**
+ * reason of incapable
+ */
+ public IncapableCause incapableCause;
+
+ /**
* Marker objects to indicate all special features
* (dimension-as-measure, topN etc.) that have influenced the capability check.
*/
public List<CapabilityInfluence> influences = Lists.newArrayListWithCapacity(1);
public static interface CapabilityInfluence {
- /** Suggest a multiplier to influence query cost */
+ /**
+ * Suggest a multiplier to influence query cost
+ */
double suggestCostMultiplier();
MeasureDesc getInvolvedMeasure();
@@ -68,4 +81,58 @@ public class CapabilityResult {
return function;
}
}
+
+ public static enum IncapableType {
+ UNMATCHED_DIMENSION, UNMATCHED_AGGREGATION, UNSUPPORT_MASSIN, UNSUPPORT_RAWQUERY, LIMIT_PRECEDE_AGGR, II_UNMATCHED_FACT_TABLE, II_MISSING_COLS
+ }
+
+ public static class IncapableCause {
+ private IncapableType incapableType;
+ private Collection<TblColRef> unmatchedDimensions;
+ private Collection<FunctionDesc> unmatchedAggregations;
+
+ public static IncapableCause unmatchedDimensions(Collection<TblColRef> unmatchedDimensions) {
+ IncapableCause incapableCause = new IncapableCause();
+ incapableCause.setIncapableType(IncapableType.UNMATCHED_DIMENSION);
+ incapableCause.setUnmatchedDimensions(unmatchedDimensions);
+ return incapableCause;
+ }
+
+ public static IncapableCause unmatchedAggregations(Collection<FunctionDesc> unmatchedAggregations) {
+ IncapableCause incapableCause = new IncapableCause();
+ incapableCause.setIncapableType(IncapableType.UNMATCHED_AGGREGATION);
+ incapableCause.setUnmatchedAggregations(unmatchedAggregations);
+ return incapableCause;
+ }
+
+ public static IncapableCause create(IncapableType incapableType) {
+ IncapableCause incapableCause = new IncapableCause();
+ incapableCause.setIncapableType(incapableType);
+ return incapableCause;
+ }
+
+ public IncapableType getIncapableType() {
+ return incapableType;
+ }
+
+ public void setIncapableType(IncapableType incapableType) {
+ this.incapableType = incapableType;
+ }
+
+ public Collection<TblColRef> getUnmatchedDimensions() {
+ return unmatchedDimensions;
+ }
+
+ public void setUnmatchedDimensions(Collection<TblColRef> unmatchedDimensions) {
+ this.unmatchedDimensions = unmatchedDimensions;
+ }
+
+ public Collection<FunctionDesc> getUnmatchedAggregations() {
+ return unmatchedAggregations;
+ }
+
+ public void setUnmatchedAggregations(Collection<FunctionDesc> unmatchedAggregations) {
+ this.unmatchedAggregations = unmatchedAggregations;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 1b113ee..df68e10 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -188,6 +188,8 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
result.capable = true;
result.cost = Math.min(result.cost, child.cost);
result.influences.addAll(child.influences);
+ } else {
+ result.incapableCause = child.incapableCause;
}
}
@@ -220,7 +222,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
init();
return allColumnDescs;
}
-
+
@Override
public List<MeasureDesc> getMeasures() {
init();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index d1608e9..c5c4c44 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -41,6 +41,7 @@ import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.realization.SQLDigest.SQLCall;
import org.apache.kylin.metadata.tuple.TupleInfo;
+import org.apache.kylin.query.routing.RealizationCheck;
import org.apache.kylin.query.schema.OLAPSchema;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hybrid.HybridInstance;
@@ -124,6 +125,7 @@ public class OLAPContext {
// cube metadata
public IRealization realization;
+ public RealizationCheck realizationCheck;
public Set<TblColRef> allColumns = new HashSet<>();
public List<TblColRef> groupByColumns = new ArrayList<>();
@@ -161,8 +163,7 @@ public class OLAPContext {
metricsColumns, aggregations, aggrSqlCalls, // aggregation
filterColumns, filter, havingFilter, // filter
sortColumns, sortOrders, limitPrecedesAggr, // sort & limit
- involvedMeasure
- );
+ involvedMeasure);
return sqlDigest;
}
@@ -180,7 +181,7 @@ public class OLAPContext {
return true;
}
}
-
+
return false;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 3b5c3cf..5fa34f7 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -90,7 +90,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
final JoinInfo joinInfo = JoinInfo.of(left, right, condition);
assert joinInfo.isEqui();
try {
- return new OLAPJoinRel(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys, variablesSet, joinType);
+ return new OLAPJoinRel(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys,
+ joinInfo.rightKeys, variablesSet, joinType);
} catch (InvalidRelException e) {
// Semantic error not possible. Must be a bug. Convert to internal error.
throw new AssertionError(e);
@@ -175,11 +176,14 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
this.context.allColumns.clear();
// build JoinDesc
+ Preconditions.checkState(this.getCondition() instanceof RexCall, "Cartesian Join is not supported.");
+
RexCall condition = (RexCall) this.getCondition();
JoinDesc join = buildJoin(condition);
JoinRelType joinRelType = this.getJoinType();
- String joinType = joinRelType == JoinRelType.INNER ? "INNER" : joinRelType == JoinRelType.LEFT ? "LEFT" : null;
+ String joinType = joinRelType == JoinRelType.INNER ? "INNER"
+ : joinRelType == JoinRelType.LEFT ? "LEFT" : null;
join.setType(joinType);
this.context.joins.add(join);
@@ -210,7 +214,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
columns.addAll(rightColumnRowType.getAllColumns());
if (columns.size() != this.rowType.getFieldCount()) {
- throw new IllegalStateException("RowType=" + this.rowType.getFieldCount() + ", ColumnRowType=" + columns.size());
+ throw new IllegalStateException(
+ "RowType=" + this.rowType.getFieldCount() + ", ColumnRowType=" + columns.size());
}
return new ColumnRowType(columns);
}
@@ -308,7 +313,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
RelOptTable factTable = context.firstTableScan.getTable();
- MethodCallExpression exprCall = Expressions.call(factTable.getExpression(OLAPTable.class), "executeOLAPQuery", implementor.getRootExpression(), Expressions.constant(context.id));
+ MethodCallExpression exprCall = Expressions.call(factTable.getExpression(OLAPTable.class), "executeOLAPQuery",
+ implementor.getRootExpression(), Expressions.constant(context.id));
return implementor.result(physType, Blocks.toBlock(exprCall));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
index dbf69bc..e96eee8 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
@@ -41,7 +41,8 @@ public class QueryRouter {
private static final Logger logger = LoggerFactory.getLogger(QueryRouter.class);
- public static IRealization selectRealization(OLAPContext olapContext, Set<IRealization> realizations) throws NoRealizationFoundException {
+ public static IRealization selectRealization(OLAPContext olapContext, Set<IRealization> realizations)
+ throws NoRealizationFoundException {
String factTableName = olapContext.firstTableScan.getTableName();
String projectName = olapContext.olapSchema.getProjectName();
@@ -53,11 +54,16 @@ public class QueryRouter {
candidates.add(new Candidate(real, sqlDigest));
}
- logger.info("Find candidates by table " + factTableName + " and project=" + projectName + " : " + StringUtils.join(candidates, ","));
+ logger.info("Find candidates by table " + factTableName + " and project=" + projectName + " : "
+ + StringUtils.join(candidates, ","));
+
+ List<Candidate> originCandidates = Lists.newArrayList(candidates);
// rule based realization selection, rules might reorder realizations or remove specific realization
RoutingRule.applyRules(candidates);
+ collectIncapableReason(olapContext, originCandidates);
+
if (candidates.size() == 0) {
return null;
}
@@ -65,7 +71,8 @@ public class QueryRouter {
Candidate chosen = candidates.get(0);
adjustForDimensionAsMeasure(chosen, olapContext);
- logger.info("The realizations remaining: " + RoutingRule.getPrintableText(candidates) + " And the final chosen one is the first one");
+ logger.info("The realizations remaining: " + RoutingRule.getPrintableText(candidates)
+ + " And the final chosen one is the first one");
for (CapabilityInfluence influence : chosen.getCapability().influences) {
if (influence.getInvolvedMeasure() != null) {
@@ -88,4 +95,16 @@ public class QueryRouter {
}
}
+ private static void collectIncapableReason(OLAPContext olapContext, List<Candidate> candidates) {
+ for (Candidate candidate : candidates) {
+ if (!candidate.getCapability().capable) {
+ RealizationCheck.IncapableReason reason = RealizationCheck.IncapableReason
+ .create(candidate.getCapability().incapableCause);
+ if (reason != null)
+ olapContext.realizationCheck.addIncapableCube(candidate.getRealization(), reason);
+ } else {
+ olapContext.realizationCheck.addCapableCube(candidate.getRealization());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/routing/RealizationCheck.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RealizationCheck.java b/query/src/main/java/org/apache/kylin/query/routing/RealizationCheck.java
new file mode 100644
index 0000000..7556b83
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/routing/RealizationCheck.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.routing;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.CapabilityResult;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.query.relnode.OLAPTableScan;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+public class RealizationCheck {
+ private Map<DataModelDesc, List<IncapableReason>> modelIncapableReasons = Maps.newHashMap();
+ private Map<CubeDesc, IncapableReason> cubeIncapableReasons = Maps.newHashMap();
+ private Map<CubeDesc, Boolean> cubeCapabilities = Maps.newHashMap();
+ private List<DataModelDesc> capableModels = Lists.newArrayList();
+
+ public Map<DataModelDesc, List<IncapableReason>> getModelIncapableReasons() {
+ return modelIncapableReasons;
+ }
+
+ public Map<CubeDesc, IncapableReason> getCubeIncapableReasons() {
+ return cubeIncapableReasons;
+ }
+
+ public Map<CubeDesc, Boolean> getCubeCapabilities() {
+ return cubeCapabilities;
+ }
+
+ public void addCapableCube(IRealization realization) {
+ if (realization instanceof CubeInstance) {
+ cubeCapabilities.put(((CubeInstance) realization).getDescriptor(), true);
+ }
+ }
+
+ public void addIncapableCube(IRealization realization) {
+ if (realization instanceof CubeInstance) {
+ cubeCapabilities.put(((CubeInstance) realization).getDescriptor(), false);
+ }
+ }
+
+ public void addIncapableCube(IRealization realization, IncapableReason incapableReason) {
+ if (realization instanceof CubeInstance) {
+ cubeCapabilities.put(((CubeInstance) realization).getDescriptor(), false);
+ cubeIncapableReasons.put(((CubeInstance) realization).getDescriptor(), incapableReason);
+ }
+ }
+
+ public void addCubeIncapableReason(IRealization realization, IncapableReason incapableReason) {
+ if (realization instanceof CubeInstance) {
+ cubeIncapableReasons.put(((CubeInstance) realization).getDescriptor(), incapableReason);
+ }
+ }
+
+ public List<DataModelDesc> getCapableModels() {
+ return capableModels;
+ }
+
+ public void addModelIncapableReason(DataModelDesc modelDesc, IncapableReason reason) {
+ if (!modelIncapableReasons.containsKey(modelDesc)) {
+ List<IncapableReason> reasons = Lists.newArrayList(reason);
+ modelIncapableReasons.put(modelDesc, reasons);
+ } else {
+ modelIncapableReasons.get(modelDesc).add(reason);
+ }
+ }
+
+ public void addCapableModel(DataModelDesc modelDesc) {
+ this.capableModels.add(modelDesc);
+ }
+
+ public void addModelIncapableReason(DataModelDesc modelDesc, List<IncapableReason> reasons) {
+ modelIncapableReasons.put(modelDesc, reasons);
+ }
+
+ public static enum IncapableType {
+ CUBE_NOT_READY, CUBE_NOT_CONTAIN_TABLE, CUBE_NOT_CONTAIN_ALL_COLUMN, CUBE_NOT_CONTAIN_ALL_DIMENSION, CUBE_NOT_CONTAIN_ALL_MEASURE, CUBE_BLACK_OUT_REALIZATION, CUBE_UN_SUPPORT_MASSIN, CUBE_UN_SUPPORT_RAWQUERY, CUBE_UNMATCHED_DIMENSION, CUBE_LIMIT_PRECEDE_AGGR, CUBE_UNMATCHED_AGGREGATION, CUBE_OTHER_CUBE_INCAPABLE, MODEL_UNMATCHED_JOIN, MODEL_JOIN_TYPE_UNMATCHED, MODEL_JOIN_CONDITION_UNMATCHED, MODEL_JOIN_NOT_FOUND, MODEL_BAD_JOIN_SEQUENCE, MODEL_FACT_TABLE_NOT_FOUND, MODEL_OTHER_MODEL_INCAPABLE
+ }
+
+ public static class IncapableReason {
+ private IncapableType incapableType;
+ // notFoundColumns = notFoundDimensions + notFoundMeasures;
+ private Collection<TblColRef> notFoundColumns;
+ private Collection<TblColRef> notFoundDimensions;
+ private Collection<TblColRef> notFoundMeasures;
+ private Collection<TblColRef> unmatchedDimensions;
+ private Collection<FunctionDesc> unmatchedAggregations;
+ private Collection<OLAPTableScan> notFoundTables;
+
+ public static IncapableReason create(IncapableType incapableType) {
+ IncapableReason incapableReason = new IncapableReason();
+ incapableReason.setIncapableType(incapableType);
+ return incapableReason;
+ }
+
+ public static IncapableReason create(CapabilityResult.IncapableCause incapableCause) {
+ if (incapableCause == null) {
+ return null;
+ }
+ IncapableReason incapableReason = new IncapableReason();
+ IncapableType incapableType = null;
+ switch (incapableCause.getIncapableType()) {
+ case UNSUPPORT_MASSIN:
+ incapableType = IncapableType.CUBE_UN_SUPPORT_MASSIN;
+ break;
+ case UNMATCHED_DIMENSION:
+ incapableType = IncapableType.CUBE_UNMATCHED_DIMENSION;
+ break;
+ case LIMIT_PRECEDE_AGGR:
+ incapableType = IncapableType.CUBE_LIMIT_PRECEDE_AGGR;
+ break;
+ case UNMATCHED_AGGREGATION:
+ incapableType = IncapableType.CUBE_UNMATCHED_AGGREGATION;
+ break;
+ case UNSUPPORT_RAWQUERY:
+ incapableType = IncapableType.CUBE_UN_SUPPORT_RAWQUERY;
+ break;
+ case II_UNMATCHED_FACT_TABLE:
+ incapableType = IncapableType.MODEL_FACT_TABLE_NOT_FOUND;
+ break;
+ case II_MISSING_COLS:
+ incapableType = IncapableType.CUBE_NOT_CONTAIN_ALL_COLUMN;
+ break;
+ default:
+ break;
+ }
+ incapableReason.setIncapableType(incapableType);
+ incapableReason.setUnmatchedDimensions(incapableCause.getUnmatchedDimensions());
+ incapableReason.setUnmatchedAggregations(incapableCause.getUnmatchedAggregations());
+ return incapableReason;
+ }
+
+ public static IncapableReason notContainAllColumn(Collection<TblColRef> notFoundColumns) {
+ IncapableReason incapableReason = new IncapableReason();
+ incapableReason.setIncapableType(IncapableType.CUBE_NOT_CONTAIN_ALL_COLUMN);
+ incapableReason.setNotFoundColumns(notFoundColumns);
+ return incapableReason;
+ }
+
+ public static IncapableReason notContainAllDimension(Collection<TblColRef> notFoundDimensions) {
+ IncapableReason incapableReason = new IncapableReason();
+ incapableReason.setIncapableType(IncapableType.CUBE_NOT_CONTAIN_ALL_DIMENSION);
+ incapableReason.setNotFoundDimensions(notFoundDimensions);
+ return incapableReason;
+ }
+
+ public static IncapableReason notContainAllMeasures(Collection<TblColRef> notFoundMeasures) {
+ IncapableReason incapableReason = new IncapableReason();
+ incapableReason.setIncapableType(IncapableType.CUBE_NOT_CONTAIN_ALL_MEASURE);
+ incapableReason.setNotFoundMeasures(notFoundMeasures);
+ return incapableReason;
+ }
+
+ public static IncapableReason notFoundTables(Collection<OLAPTableScan> notFoundTables) {
+ IncapableReason incapableReason = new IncapableReason();
+ incapableReason.setIncapableType(IncapableType.CUBE_NOT_CONTAIN_TABLE);
+ incapableReason.setNotFoundTables(notFoundTables);
+ return incapableReason;
+ }
+
+ public void setIncapableType(IncapableType incapableType) {
+ this.incapableType = incapableType;
+ }
+
+ public void setUnmatchedDimensions(Collection<TblColRef> unmatchedDimensions) {
+ this.unmatchedDimensions = unmatchedDimensions;
+ }
+
+ public void setUnmatchedAggregations(Collection<FunctionDesc> unmatchedAggregations) {
+ this.unmatchedAggregations = unmatchedAggregations;
+ }
+
+ public void setNotFoundColumns(Collection<TblColRef> notFoundColumns) {
+ this.notFoundColumns = notFoundColumns;
+ }
+
+ public void setNotFoundTables(Collection<OLAPTableScan> notFoundTables) {
+ this.notFoundTables = notFoundTables;
+ }
+
+ public void setNotFoundDimensions(Collection<TblColRef> notFoundDimensions) {
+ this.notFoundDimensions = notFoundDimensions;
+ }
+
+ public void setNotFoundMeasures(Collection<TblColRef> notFoundMeasures) {
+ this.notFoundMeasures = notFoundMeasures;
+ }
+
+ public Collection<TblColRef> getNotFoundDimensions() {
+ return notFoundDimensions;
+ }
+
+ public Collection<TblColRef> getNotFoundMeasures() {
+ return notFoundMeasures;
+ }
+
+ public IncapableType getIncapableType() {
+ return incapableType;
+ }
+
+ public Collection<TblColRef> getUnmatchedDimensions() {
+ return unmatchedDimensions;
+ }
+
+ public Collection<TblColRef> getNotFoundColumns() {
+ return notFoundColumns;
+ }
+
+ public Collection<FunctionDesc> getUnmatchedAggregations() {
+ return unmatchedAggregations;
+ }
+
+ public Collection<OLAPTableScan> getNotFoundTables() {
+ return notFoundTables;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java b/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
index cfb0cbd..56580c6 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
@@ -43,6 +43,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -53,7 +54,9 @@ public class RealizationChooser {
// select models for given contexts, return realization candidates for each context
public static void selectRealization(List<OLAPContext> contexts) {
// try different model for different context
+
for (OLAPContext ctx : contexts) {
+ ctx.realizationCheck = new RealizationCheck();
attemptSelectRealization(ctx);
Preconditions.checkNotNull(ctx.realization);
}
@@ -109,6 +112,8 @@ public class RealizationChooser {
matchUp = ImmutableMap.of(firstTable.getAlias(), modelAlias);
} else if (ctx.joins.size() != ctx.allTableScans.size() - 1) {
// has hanging tables
+ ctx.realizationCheck.addModelIncapableReason(model,
+ RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.MODEL_BAD_JOIN_SEQUENCE));
throw new IllegalStateException("Please adjust the sequence of join tables. " + toErrorMsg(ctx));
} else {
// normal big joins
@@ -118,9 +123,13 @@ public class RealizationChooser {
matchUp = ctx.joinsTree.matches(model.getJoinsTree(), result);
}
- if (matchUp == null)
+ if (matchUp == null) {
+ ctx.realizationCheck.addModelIncapableReason(model,
+ RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.MODEL_UNMATCHED_JOIN));
return null;
+ }
+ ctx.realizationCheck.addCapableModel(model);
result.putAll(matchUp);
return result;
@@ -131,17 +140,28 @@ public class RealizationChooser {
KylinConfig kylinConfig = first.olapSchema.getConfig();
String projectName = first.olapSchema.getProjectName();
String factTableName = first.firstTableScan.getOlapTable().getTableName();
- Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, factTableName);
+ Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName,
+ factTableName);
final Map<DataModelDesc, Set<IRealization>> models = Maps.newHashMap();
final Map<DataModelDesc, RealizationCost> costs = Maps.newHashMap();
+
for (IRealization real : realizations) {
- if (real.isReady() == false)
+ if (real.isReady() == false) {
+ context.realizationCheck.addIncapableCube(real,
+ RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.CUBE_NOT_READY));
continue;
- if (containsAll(real.getAllColumnDescs(), first.allColumns) == false)
+ }
+ if (containsAll(real.getAllColumnDescs(), first.allColumns) == false) {
+ context.realizationCheck.addIncapableCube(real, RealizationCheck.IncapableReason
+ .notContainAllColumn(notContain(real.getAllColumnDescs(), first.allColumns)));
continue;
- if (RemoveBlackoutRealizationsRule.accept(real) == false)
+ }
+ if (RemoveBlackoutRealizationsRule.accept(real) == false) {
+ context.realizationCheck.addIncapableCube(real, RealizationCheck.IncapableReason
+ .create(RealizationCheck.IncapableType.CUBE_BLACK_OUT_REALIZATION));
continue;
+ }
RealizationCost cost = new RealizationCost(real);
DataModelDesc m = real.getModel();
@@ -184,6 +204,15 @@ public class RealizationChooser {
return true;
}
+ private static List<TblColRef> notContain(Set<ColumnDesc> allColumnDescs, Set<TblColRef> allColumns) {
+ List<TblColRef> notContainCols = Lists.newArrayList();
+ for (TblColRef col : allColumns) {
+ if (!allColumnDescs.contains(col.getColumnDesc()))
+ notContainCols.add(col);
+ }
+ return notContainCols;
+ }
+
private static void fixModel(OLAPContext context, DataModelDesc model, Map<String, String> aliasMap) {
for (OLAPTableScan tableScan : context.allTableScans) {
tableScan.fixColumnRowTypeWithModel(model, aliasMap);
@@ -205,7 +234,8 @@ public class RealizationChooser {
this.priority = Candidate.PRIORITIES.get(real.getType());
// ref CubeInstance.getCost()
- int c = real.getAllDimensions().size() * CubeInstance.COST_WEIGHT_DIMENSION + real.getMeasures().size() * CubeInstance.COST_WEIGHT_MEASURE;
+ int c = real.getAllDimensions().size() * CubeInstance.COST_WEIGHT_DIMENSION
+ + real.getMeasures().size() * CubeInstance.COST_WEIGHT_MEASURE;
for (JoinTableDesc join : real.getModel().getJoinTables()) {
if (join.getJoin().isInnerJoin())
c += CubeInstance.COST_WEIGHT_INNER_JOIN;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java
index 576b47f..56e6eb5 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java
@@ -34,9 +34,8 @@ public class RemoveUncapableRealizationsRule extends RoutingRule {
Candidate candidate = iterator.next();
CapabilityResult capability = candidate.getRealization().isCapable(candidate.getSqlDigest());
- if (capability.capable)
- candidate.setCapability(capability);
- else
+ candidate.setCapability(capability);
+ if (!capability.capable)
iterator.remove();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index d46d4ff..377ca89 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -186,10 +186,10 @@ public class QueryUtil {
errorMsg = errorMsg.replaceAll("\\s", " ");
// move cause to be ahead of sql, calcite creates the message pattern below
- Pattern pattern = Pattern.compile("error while executing SQL \"(.*)\":(.*)");
+ Pattern pattern = Pattern.compile("Error while executing SQL \"(.*)\":(.*)");
Matcher matcher = pattern.matcher(errorMsg);
if (matcher.find()) {
- return matcher.group(2).trim() + "\n" + "while executing SQL: \"" + matcher.group(1).trim() + "\"";
+ return matcher.group(2).trim() + "\nwhile executing SQL: \"" + matcher.group(1).trim() + "\"";
} else
return errorMsg;
} catch (Exception e) {
@@ -201,7 +201,8 @@ public class QueryUtil {
String sql1 = sql.toLowerCase();
sql1 = removeCommentInSql(sql1);
sql1 = sql1.trim();
- return sql1.startsWith("select") || (sql1.startsWith("with") && sql1.contains("select")) || (sql1.startsWith("explain") && sql1.contains("select"));
+ return sql1.startsWith("select") || (sql1.startsWith("with") && sql1.contains("select"))
+ || (sql1.startsWith("explain") && sql1.contains("select"));
}
public static String removeCommentInSql(String sql1) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 2b1626f..088ed6d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -456,7 +456,7 @@ public class QueryService extends BasicService {
} catch (Throwable e) { // calcite may throw AssertError
logger.error("Exception while executing query", e);
- String errMsg = QueryUtil.makeErrorMsgUserFriendly(e);
+ String errMsg = makeErrorMsgUserFriendly(e);
sqlResponse = new SQLResponse(null, null, 0, true, errMsg);
sqlResponse.setTotalScanCount(queryContext.getScannedRows());
@@ -871,6 +871,10 @@ public class QueryService extends BasicService {
return buildSqlResponse(isPushDown, results, columnMetas);
}
+ protected String makeErrorMsgUserFriendly(Throwable e) {
+ return QueryUtil.makeErrorMsgUserFriendly(e);
+ }
+
private SQLResponse getPrepareOnlySqlResponse(String correctedSql, Connection conn, Boolean isPushDown,
List<List<String>> results, List<SelectedColumnMeta> columnMetas) throws SQLException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b97e16ad/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
index c9f9296..7c5f253 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
@@ -69,10 +69,11 @@ public class QueryControllerTest extends ServiceTestBase {
@Test
public void testErrorMsg() {
- String errorMsg = "error while executing SQL \"select lkp.clsfd_ga_prfl_id, ga.sum_dt, sum(ga.bounces) as bounces, sum(ga.exits) as exits, sum(ga.entrances) as entrances, sum(ga.pageviews) as pageviews, count(distinct ga.GA_VSTR_ID, ga.GA_VST_ID) as visits, count(distinct ga.GA_VSTR_ID) as uniqVistors from CLSFD_GA_PGTYPE_CATEG_LOC ga left join clsfd_ga_prfl_lkp lkp on ga.SRC_GA_PRFL_ID = lkp.SRC_GA_PRFL_ID group by lkp.clsfd_ga_prfl_id,ga.sum_dt order by lkp.clsfd_ga_prfl_id,ga.sum_dt LIMIT 50000\": From line 14, column 14 to line 14, column 29: Column 'CLSFD_GA_PRFL_ID' not found in table 'LKP'";
- assert QueryUtil.makeErrorMsgUserFriendly(errorMsg).equals(
+ String errorMsg = "Error while executing SQL \"select lkp.clsfd_ga_prfl_id, ga.sum_dt, sum(ga.bounces) as bounces, sum(ga.exits) as exits, sum(ga.entrances) as entrances, sum(ga.pageviews) as pageviews, count(distinct ga.GA_VSTR_ID, ga.GA_VST_ID) as visits, count(distinct ga.GA_VSTR_ID) as uniqVistors from CLSFD_GA_PGTYPE_CATEG_LOC ga left join clsfd_ga_prfl_lkp lkp on ga.SRC_GA_PRFL_ID = lkp.SRC_GA_PRFL_ID group by lkp.clsfd_ga_prfl_id,ga.sum_dt order by lkp.clsfd_ga_prfl_id,ga.sum_dt LIMIT 50000\": From line 14, column 14 to line 14, column 29: Column 'CLSFD_GA_PRFL_ID' not found in table 'LKP'";
+ Assert.assertEquals(
"From line 14, column 14 to line 14, column 29: Column 'CLSFD_GA_PRFL_ID' not found in table 'LKP'\n"
- + "while executing SQL: \"select lkp.clsfd_ga_prfl_id, ga.sum_dt, sum(ga.bounces) as bounces, sum(ga.exits) as exits, sum(ga.entrances) as entrances, sum(ga.pageviews) as pageviews, count(distinct ga.GA_VSTR_ID, ga.GA_VST_ID) as visits, count(distinct ga.GA_VSTR_ID) as uniqVistors from CLSFD_GA_PGTYPE_CATEG_LOC ga left join clsfd_ga_prfl_lkp lkp on ga.SRC_GA_PRFL_ID = lkp.SRC_GA_PRFL_ID group by lkp.clsfd_ga_prfl_id,ga.sum_dt order by lkp.clsfd_ga_prfl_id,ga.sum_dt LIMIT 50000\"");
+ + "while executing SQL: \"select lkp.clsfd_ga_prfl_id, ga.sum_dt, sum(ga.bounces) as bounces, sum(ga.exits) as exits, sum(ga.entrances) as entrances, sum(ga.pageviews) as pageviews, count(distinct ga.GA_VSTR_ID, ga.GA_VST_ID) as visits, count(distinct ga.GA_VSTR_ID) as uniqVistors from CLSFD_GA_PGTYPE_CATEG_LOC ga left join clsfd_ga_prfl_lkp lkp on ga.SRC_GA_PRFL_ID = lkp.SRC_GA_PRFL_ID group by lkp.clsfd_ga_prfl_id,ga.sum_dt order by lkp.clsfd_ga_prfl_id,ga.sum_dt LIMIT 50000\"",
+ QueryUtil.makeErrorMsgUserFriendly(errorMsg));
}
@Test