You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/10/26 23:33:38 UTC
[1/3] kylin git commit: KYLIN-2127 UI bug fix for Extend Column
[Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-1971 8974b99af -> 293eb85b6 (forced update)
KYLIN-2127 UI bug fix for Extend Column
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/577052bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/577052bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/577052bd
Branch: refs/heads/KYLIN-1971
Commit: 577052bd2246ec1feb02771f11ca2642a8076c4b
Parents: 5cdd225
Author: kangkaisen <ka...@live.com>
Authored: Tue Oct 25 22:27:54 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Wed Oct 26 19:13:45 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeMeasures.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/577052bd/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index dcf26b7..938e918 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -229,7 +229,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
$scope.nextPara = {
"type":"column",
"value":"",
- "next_parameter":{}
+ "next_parameter": null
}
if($scope.newMeasure){
$scope.newMeasure.function.parameter.next_parameter = null;
[2/3] kylin git commit: minor,
fix concurrent issue with CubeSegment.cuboidBaseShards
Posted by li...@apache.org.
minor, fix concurrent issue with CubeSegment.cuboidBaseShards
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/69213e99
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/69213e99
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/69213e99
Branch: refs/heads/KYLIN-1971
Commit: 69213e9991a1a69b26803852193a78628246b127
Parents: 577052b
Author: gaodayue <ga...@meituan.com>
Authored: Wed Oct 26 21:39:11 2016 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Wed Oct 26 21:39:11 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/cube/CubeSegment.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/69213e99/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index fdf1fb0..b3d6d6b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -49,6 +49,8 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import javax.annotation.concurrent.GuardedBy;
+
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegment {
@@ -115,7 +117,8 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private Map<String, String> additionalInfo = new LinkedHashMap<String, String>();
- private volatile Map<Long, Short> cuboidBaseShards = Maps.newHashMap();//cuboid id ==> base(starting) shard for this cuboid
+ @GuardedBy("this")
+ private Map<Long, Short> cuboidBaseShards = Maps.newHashMap(); // cuboid id ==> base(starting) shard for this cuboid
public CubeDesc getCubeDesc() {
return getCubeInstance().getDescriptor();
@@ -516,7 +519,7 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
this.totalShards = totalShards;
}
- public short getCuboidBaseShard(Long cuboidId) {
+ public synchronized short getCuboidBaseShard(Long cuboidId) {
if (totalShards > 0) {
//shard squashed case
[3/3] kylin git commit: TblColRef != ColumnDesc, turns on
Posted by li...@apache.org.
TblColRef != ColumnDesc, turns on
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/293eb85b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/293eb85b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/293eb85b
Branch: refs/heads/KYLIN-1971
Commit: 293eb85b63869f25d356309b28a0b6a3a55aad06
Parents: 69213e9
Author: Li Yang <li...@apache.org>
Authored: Wed Oct 26 18:24:15 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Thu Oct 27 07:15:53 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 6 +
.../org/apache/kylin/cube/model/CubeDesc.java | 14 ++-
.../apache/kylin/metadata/model/JoinDesc.java | 2 +-
.../apache/kylin/metadata/model/TblColRef.java | 4 +-
.../metadata/realization/IRealization.java | 3 +
.../kylin/storage/hybrid/HybridInstance.java | 17 +++
.../apache/kylin/query/ITKylinQueryTest.java | 2 +-
.../relnode/OLAPToEnumerableConverter.java | 53 +++++----
.../kylin/query/routing/ModelChooser.java | 111 ++++++++++++-------
9 files changed, 139 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 2ccdde7..6bd3826 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.metadata.model.LookupDesc;
@@ -395,6 +396,11 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
@Override
+ public Set<ColumnDesc> getAllColumnDescs() {
+ return getDescriptor().listAllColumnDescs();
+ }
+
+ @Override
public long getDateRangeStart() {
List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 5c73f21..0c0b4f3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -54,6 +54,7 @@ import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IEngineAware;
@@ -159,8 +160,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
@JsonProperty("override_kylin_properties")
private LinkedHashMap<String, String> overrideKylinProps = new LinkedHashMap<String, String>();
- private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>();
- private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<TblColRef>();
+ private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>();
+ private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>();
+ private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>();
private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap();
private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap();
@@ -191,6 +193,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
public Set<TblColRef> listAllColumns() {
return allColumns;
}
+
+ public Set<ColumnDesc> listAllColumnDescs() {
+ return allColumnDescs;
+ }
/**
* @return dimension columns including derived, BUT NOT measures
@@ -549,6 +555,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey columns count (%d) doesn't match dimensions columns count (%d)", rowkey.getRowKeyColumns().length, dimCols.size());
initDictionaryDesc();
+
+ for (TblColRef col : allColumns) {
+ allColumnDescs.add(col.getColumnDesc());
+ }
}
public void validateAggregationGroups() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index 04fbf62..5beda0a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -154,7 +154,7 @@ public class JoinDesc {
int match = 0;
for (int i = 0; i < a.length; i++) {
for (int j = 0; j < b.length; j++) {
- if (a[i].equals(b[j])) {
+ if (a[i].getColumnDesc().equals(b[j].getColumnDesc())) {
match++;
break;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 8841ee5..f44972c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -183,8 +183,8 @@ public class TblColRef implements Serializable {
return false;
if (!StringUtils.equals(column.getName(), other.column.getName()))
return false;
-// if ((table == null ? other.table == null : table.equals(other.table)) == false)
-// return false;
+ if ((table == null ? other.table == null : table.equals(other.table)) == false)
+ return false;
return true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index 343ec99..f1770d7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -43,6 +44,8 @@ public interface IRealization extends IStorageAware {
public String getFactTable();
public Set<TblColRef> getAllColumns();
+
+ public Set<ColumnDesc> getAllColumnDescs();
public List<TblColRef> getAllDimensions();
http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 57d14d5..d7007ac 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -68,6 +69,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
private volatile IRealization[] realizations = null;
private List<TblColRef> allDimensions = null;
private Set<TblColRef> allColumns = null;
+ private Set<ColumnDesc> allColumnDescs = null;
private List<MeasureDesc> allMeasures = null;
private long dateRangeStart;
private long dateRangeEnd;
@@ -138,6 +140,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
allDimensions = Lists.newArrayList(dimensions);
allColumns = columns;
+ allColumnDescs = asColumnDescs(allColumns);
allMeasures = Lists.newArrayList(measures);
Collections.sort(realizationList, new Comparator<IRealization>() {
@@ -166,6 +169,14 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
}
}
+ private Set<ColumnDesc> asColumnDescs(Set<TblColRef> columns) {
+ LinkedHashSet<ColumnDesc> result = new LinkedHashSet<>();
+ for (TblColRef col : columns) {
+ result.add(col.getColumnDesc());
+ }
+ return result;
+ }
+
@Override
public CapabilityResult isCapable(SQLDigest digest) {
CapabilityResult result = new CapabilityResult();
@@ -210,6 +221,12 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
}
@Override
+ public Set<ColumnDesc> getAllColumnDescs() {
+ init();
+ return allColumnDescs;
+ }
+
+ @Override
public List<MeasureDesc> getMeasures() {
init();
return allMeasures;
http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 2ec5324..2065bff 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -154,7 +154,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql";
+ String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_subquery/query11.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 24fc430..31cfc36 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -48,10 +48,10 @@ import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.routing.ModelChooser;
-import org.apache.kylin.query.routing.NoRealizationFoundException;
import org.apache.kylin.query.routing.QueryRouter;
import org.apache.kylin.query.schema.OLAPTable;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
/**
@@ -79,30 +79,23 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
OLAPRel.OLAPImplementor olapImplementor = new OLAPRel.OLAPImplementor();
olapImplementor.visitChild(getInput(), this);
- // find cube from olap context and apply cell level security
- try {
- for (OLAPContext context : OLAPContext.getThreadLocalContexts()) {
- // Context has no table scan is created by OLAPJoinRel which looks like
- // (sub-query) as A join (sub-query) as B
- // No realization needed for such context.
- if (context.firstTableScan == null) {
- continue;
- }
-
- Set<IRealization> candidates = ModelChooser.selectModel(context);
- IRealization realization = QueryRouter.selectRealization(context, candidates);
- context.realization = realization;
-
- doAccessControl(context);
- }
- } catch (NoRealizationFoundException e) {
- OLAPContext ctx0 = (OLAPContext) OLAPContext.getThreadLocalContexts().toArray()[0];
- if (ctx0 != null && ctx0.olapSchema.hasStarSchemaUrl()) {
- // generate hive result
- return buildHiveResult(enumImplementor, pref, ctx0);
- } else {
- throw e;
+ // identify model
+ List<OLAPContext> contexts = listContextsHavingScan();
+ Set<IRealization> candidates = ModelChooser.selectModel(contexts);
+
+ // identify realization for each context
+ for (OLAPContext context : OLAPContext.getThreadLocalContexts()) {
+
+ // Context has no table scan is created by OLAPJoinRel which looks like
+ // (sub-query) as A join (sub-query) as B
+ // No realization needed for such context.
+ if (context.firstTableScan == null) {
+ continue;
}
+
+ IRealization realization = QueryRouter.selectRealization(context, candidates);
+ context.realization = realization;
+ doAccessControl(context);
}
// rewrite query if necessary
@@ -123,6 +116,17 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
return impl.visitChild(this, 0, inputAsEnum, pref);
}
+ private List<OLAPContext> listContextsHavingScan() {
+ int size = OLAPContext.getThreadLocalContexts().size();
+ List<OLAPContext> result = Lists.newArrayListWithCapacity(size);
+ for (int i = 0; i < size; i++) {
+ OLAPContext ctx = OLAPContext.getThreadLocalContextById(i);
+ if (ctx.firstTableScan != null)
+ result.add(ctx);
+ }
+ return result;
+ }
+
private void doAccessControl(OLAPContext context) {
String controllerCls = KylinConfig.getInstanceFromEnv().getQueryAccessController();
if (null != controllerCls && !controllerCls.isEmpty()) {
@@ -176,6 +180,7 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
}
}
+ @SuppressWarnings("unused")
private Result buildHiveResult(EnumerableRelImplementor enumImplementor, Prefer pref, OLAPContext context) {
RelDataType hiveRowType = getRowType();
http://git-wip-us.apache.org/repos/asf/kylin/blob/293eb85b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index 4667f4a..08616bd 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -26,10 +26,12 @@ import java.util.TreeMap;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.LookupDesc;
import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.relnode.OLAPContext;
@@ -42,55 +44,68 @@ import com.google.common.collect.Sets;
public class ModelChooser {
- public static Set<IRealization> selectModel(OLAPContext context) {
- Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(context);
- OLAPTableScan firstTable = context.firstTableScan;
- List<JoinDesc> joins = context.joins;
+ // select a model that satisfies all the contexts
+ public static Set<IRealization> selectModel(List<OLAPContext> contexts) {
+ Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(contexts);
for (DataModelDesc model : modelMap.keySet()) {
- Map<String, String> aliasMap = matches(model, firstTable, joins);
+ Map<String, String> aliasMap = matches(model, contexts);
if (aliasMap != null) {
- fixModel(context, model, aliasMap);
+ for (OLAPContext ctx : contexts)
+ fixModel(ctx, model, aliasMap);
return modelMap.get(model);
}
}
-
- throw new NoRealizationFoundException("No model found by first table " + firstTable.getOlapTable().getTableName() + " and joins " + joins);
+
+ throw new NoRealizationFoundException("No model found for" + toErrorMsg(contexts));
}
- private static Map<String, String> matches(DataModelDesc model, OLAPTableScan firstTable, List<JoinDesc> joins) {
- Map<String, String> result = Maps.newHashMap();
-
- // no join special case
- if (joins.isEmpty()) {
- TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName());
- if (tableRef == null)
- return null;
- result.put(firstTable.getAlias(), tableRef.getAlias());
- return result;
+ private static String toErrorMsg(List<OLAPContext> contexts) {
+ StringBuilder buf = new StringBuilder();
+ for (OLAPContext ctx : contexts) {
+ buf.append(", ").append(ctx.firstTableScan);
+ for (JoinDesc join : ctx.joins)
+ buf.append(", ").append(join);
}
-
+ return buf.toString();
+ }
+
+ private static Map<String, String> matches(DataModelDesc model, List<OLAPContext> contexts) {
+ Map<String, String> result = Maps.newHashMap();
+
// the greedy match is not perfect but works for the moment
Map<String, List<JoinDesc>> modelJoinsMap = model.getJoinsMap();
- for (JoinDesc queryJoin : joins) {
- String fkTable = queryJoin.getForeignKeyColumns()[0].getTable();
- List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable);
- if (modelJoins == null)
- return null;
-
- JoinDesc matchJoin = null;
- for (JoinDesc modelJoin : modelJoins) {
- if (modelJoin.matches(queryJoin)) {
- matchJoin = modelJoin;
- break;
+ for (OLAPContext ctx : contexts) {
+ for (JoinDesc queryJoin : ctx.joins) {
+ String fkTable = queryJoin.getForeignKeyColumns()[0].getTable();
+ List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable);
+ if (modelJoins == null)
+ return null;
+
+ JoinDesc matchJoin = null;
+ for (JoinDesc modelJoin : modelJoins) {
+ if (modelJoin.matches(queryJoin)) {
+ matchJoin = modelJoin;
+ break;
+ }
}
+ if (matchJoin == null)
+ return null;
+
+ matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result);
+ matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result);
}
- if (matchJoin == null)
- return null;
- matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result);
- matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result);
+ OLAPTableScan firstTable = ctx.firstTableScan;
+ String firstTableAlias = firstTable.getAlias();
+ if (result.containsKey(firstTableAlias) == false) {
+ TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName());
+ if (tableRef == null)
+ return null;
+ matchesAdd(firstTableAlias, tableRef.getAlias(), result);
+ }
}
+
return result;
}
@@ -99,10 +114,12 @@ public class ModelChooser {
Preconditions.checkState(existingTarget == null || existingTarget.equals(targetAlias));
}
- private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(OLAPContext context) {
- KylinConfig kylinConfig = context.olapSchema.getConfig();
- String projectName = context.olapSchema.getProjectName();
- String factTableName = context.firstTableScan.getOlapTable().getTableName();
+ private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(List<OLAPContext> contexts) {
+ // the first context, which is the top most context, contains all columns from all contexts
+ OLAPContext first = contexts.get(0);
+ KylinConfig kylinConfig = first.olapSchema.getConfig();
+ String projectName = first.olapSchema.getProjectName();
+ String factTableName = first.firstTableScan.getOlapTable().getTableName();
Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, factTableName);
final Map<DataModelDesc, Set<IRealization>> models = Maps.newHashMap();
@@ -110,7 +127,7 @@ public class ModelChooser {
for (IRealization real : realizations) {
if (real.isReady() == false)
continue;
- if (real.getAllColumns().containsAll(context.allColumns) == false)
+ if (containsAll(real.getAllColumnDescs(), first.allColumns) == false)
continue;
if (RemoveBlackoutRealizationsRule.accept(real) == false)
continue;
@@ -143,20 +160,28 @@ public class ModelChooser {
return result;
}
+ private static boolean containsAll(Set<ColumnDesc> allColumnDescs, Set<TblColRef> allColumns) {
+ for (TblColRef col : allColumns) {
+ if (allColumnDescs.contains(col.getColumnDesc()) == false)
+ return false;
+ }
+ return true;
+ }
+
private static void fixModel(OLAPContext context, DataModelDesc model, Map<String, String> aliasMap) {
for (OLAPTableScan tableScan : context.allTableScans) {
tableScan.fixColumnRowTypeWithModel(model, aliasMap);
}
}
-
+
private static class RealizationCost implements Comparable<RealizationCost> {
final public int priority;
final public int cost;
-
+
public RealizationCost(IRealization real) {
// ref Candidate.PRIORITIES
this.priority = Candidate.PRIORITIES.get(real.getType());
-
+
// ref CubeInstance.getCost()
int c = real.getAllDimensions().size() * CubeInstance.COST_WEIGHT_DIMENSION + real.getMeasures().size() * CubeInstance.COST_WEIGHT_MEASURE;
for (LookupDesc lookup : real.getDataModelDesc().getLookups()) {
@@ -165,7 +190,7 @@ public class ModelChooser {
}
this.cost = c;
}
-
+
@Override
public int compareTo(RealizationCost o) {
int comp = this.priority - o.priority;