You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/10/26 03:07:46 UTC
kylin git commit: KYLIN-1971 ModelChooser,
match Model against query joins
Repository: kylin
Updated Branches:
refs/heads/master 151ec3bd1 -> 5cdd22523
KYLIN-1971 ModelChooser, match Model against query joins
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5cdd2252
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5cdd2252
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5cdd2252
Branch: refs/heads/master
Commit: 5cdd22523a68faa4879092d4b2510ea461609f6e
Parents: 151ec3b
Author: Yang Li <li...@apache.org>
Authored: Tue Oct 25 06:16:15 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Oct 26 11:06:42 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 13 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 8 +-
.../kylin/metadata/model/DataModelDesc.java | 13 +-
.../kylin/metadata/model/FunctionDesc.java | 18 --
.../apache/kylin/metadata/model/JoinDesc.java | 44 ++++-
.../apache/kylin/metadata/model/TableRef.java | 4 +-
.../apache/kylin/metadata/model/TblColRef.java | 21 ++-
.../kylin/metadata/project/ProjectL2Cache.java | 2 +-
.../metadata/realization/IRealization.java | 123 ++++++-------
.../kylin/storage/hybrid/HybridInstance.java | 7 +-
.../apache/kylin/query/ITCombinationTest.java | 9 +-
.../kylin/query/relnode/OLAPTableScan.java | 16 +-
.../relnode/OLAPToEnumerableConverter.java | 13 +-
.../kylin/query/routing/ModelChooser.java | 178 +++++++++++++++++++
.../apache/kylin/query/routing/QueryRouter.java | 6 +-
.../rules/RemoveBlackoutRealizationsRule.java | 18 +-
16 files changed, 351 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 720690d..2ccdde7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -21,6 +21,7 @@ package org.apache.kylin.cube;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
+import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
@@ -52,9 +53,9 @@ import com.google.common.collect.Lists;
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class CubeInstance extends RootPersistentEntity implements IRealization, IBuildable {
- private static final int COST_WEIGHT_MEASURE = 1;
- private static final int COST_WEIGHT_DIMENSION = 10;
- private static final int COST_WEIGHT_INNER_JOIN = 100;
+ public static final int COST_WEIGHT_MEASURE = 1;
+ public static final int COST_WEIGHT_DIMENSION = 10;
+ public static final int COST_WEIGHT_INNER_JOIN = 100;
public static CubeInstance create(String cubeName, CubeDesc cubeDesc) {
CubeInstance cubeInstance = new CubeInstance();
@@ -374,7 +375,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
for (LookupDesc lookupDesc : this.getDescriptor().getModel().getLookups()) {
// more tables, more cost
- if ("inner".equals(lookupDesc.getJoin().getType())) {
+ if (lookupDesc.getJoin().isInnerJoin()) {
// inner join cost is bigger than left join, as it will filter some records
calculatedCost += COST_WEIGHT_INNER_JOIN;
}
@@ -389,8 +390,8 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
@Override
- public List<TblColRef> getAllColumns() {
- return Lists.newArrayList(getDescriptor().listAllColumns());
+ public Set<TblColRef> getAllColumns() {
+ return getDescriptor().listAllColumns();
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 772297f..3bf0de9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -20,12 +20,11 @@ package org.apache.kylin.metadata.model;
import java.io.Serializable;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import org.apache.commons.lang.StringUtils;
import org.apache.kylin.metadata.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
@@ -154,11 +153,6 @@ public class ColumnDesc implements Serializable {
}
}
- public boolean isSameAs(String tableName, String columnName) {
- return StringUtils.equalsIgnoreCase(table.getIdentity(), tableName) && //
- StringUtils.equalsIgnoreCase(name, columnName);
- }
-
@Override
public int hashCode() {
final int prime = 31;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index a7734f1..7c39a25 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -86,6 +86,7 @@ public class DataModelDesc extends RootPersistentEntity {
private List<TableRef> lookupTableRefs = Lists.newArrayList();
private Map<String, TableRef> aliasMap = Maps.newHashMap(); // a table has exactly one alias
private Map<String, TableRef> tableNameMap = Maps.newHashMap(); // a table maybe referenced by multiple names
+ private Map<String, List<JoinDesc>> joinsMap = Maps.newHashMap();
/**
* Error messages during resolving json metadata
@@ -274,6 +275,7 @@ public class DataModelDesc extends RootPersistentEntity {
lookupTableRefs.clear();
aliasMap.clear();
tableNameMap.clear();
+ joinsMap.clear();
initTableAlias(tables);
initJoinColumns();
@@ -378,10 +380,19 @@ public class DataModelDesc extends RootPersistentEntity {
}
}
+ List<JoinDesc> list = joinsMap.get(factTableRef.getTableIdentity());
+ if (list == null)
+ joinsMap.put(factTableRef.getTableIdentity(), list = Lists.newArrayListWithCapacity(4));
+ list.add(join);
}
}
+
+ public Map<String, List<JoinDesc>> getJoinsMap() {
+ return joinsMap;
+ }
- /** * Add error info and thrown exception out
+ /**
+ * Add error info and thrown exception out
*
* @param message
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index a2c5756..e7d5186 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -19,7 +19,6 @@
package org.apache.kylin.metadata.model;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Set;
@@ -238,23 +237,6 @@ public class FunctionDesc {
this.returnDataType = DataType.getType(returnType);
}
- public TblColRef selectTblColRef(Collection<TblColRef> metricColumns, String factTableName) {
- if (this.isCount())
- return null; // count is not about any column but the whole row
-
- ParameterDesc parameter = this.getParameter();
- if (parameter == null)
- return null;
-
- String columnName = parameter.getValue();
- for (TblColRef col : metricColumns) {
- if (col.isSameAs(factTableName, columnName)) {
- return col;
- }
- }
- return null;
- }
-
public HashMap<String, String> getConfiguration() {
return configuration;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index 458367d..04fbf62 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -50,10 +50,18 @@ public class JoinDesc {
foreignKeyColumns = tt;
}
+ public boolean isInnerJoin() {
+ return "INNER".equalsIgnoreCase(type);
+ }
+
+ public boolean isLeftJoin() {
+ return "LEFT".equalsIgnoreCase(type);
+ }
+
public String getType() {
return type;
}
-
+
public void setType(String type) {
this.type = type;
}
@@ -110,9 +118,9 @@ public class JoinDesc {
return false;
JoinDesc other = (JoinDesc) obj;
- if (!this.columnsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns))
+ if (!this.colRefsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns))
return false;
- if (!this.columnsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns))
+ if (!this.colRefsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns))
return false;
if (!this.type.equalsIgnoreCase(other.getType()))
@@ -120,13 +128,41 @@ public class JoinDesc {
return true;
}
- private boolean columnsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) {
+ private boolean colRefsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) {
if (a.length != b.length)
return false;
return Arrays.asList(a).containsAll(Arrays.asList(b));
}
+ // equals() without alias
+ public boolean matches(JoinDesc other) {
+ if (!this.type.equalsIgnoreCase(other.getType()))
+ return false;
+ if (!this.columnsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns))
+ return false;
+ if (!this.columnsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns))
+ return false;
+
+ return true;
+ }
+
+ private boolean columnsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) {
+ if (a.length != b.length)
+ return false;
+
+ int match = 0;
+ for (int i = 0; i < a.length; i++) {
+ for (int j = 0; j < b.length; j++) {
+ if (a[i].equals(b[j])) {
+ match++;
+ break;
+ }
+ }
+ }
+ return match == a.length;
+ }
+
@Override
public String toString() {
return "JoinDesc [type=" + type + ", primary_key=" + Arrays.toString(primaryKey) + ", foreign_key=" + Arrays.toString(foreignKey) + "]";
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
index 186ff35..254ce18 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
@@ -86,9 +86,9 @@ public class TableRef {
TableRef t = (TableRef) o;
- if (!model.equals(t.model))
+ if ((model == null ? t.model == null : model.getName().equals(t.model.getName())) == false)
return false;
- if (!alias.equals(t.alias))
+ if ((alias == null ? t.alias == null : alias.equals(t.alias)) == false)
return false;
if (!table.getIdentity().equals(t.table.getIdentity()))
return false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 69569dd..8841ee5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -64,6 +64,9 @@ public class TblColRef implements Serializable {
}
private static final DataModelDesc UNKNOWN_MODEL = new DataModelDesc();
+ static {
+ UNKNOWN_MODEL.setName("UNKNOWN_MODEL");
+ }
public static TableRef tableForUnknownModel(String tempTableAlias, TableDesc table) {
return new TableRef(UNKNOWN_MODEL, tempTableAlias, table);
@@ -119,6 +122,10 @@ public class TblColRef implements Serializable {
return table;
}
+ public String getTableAlias() {
+ return table.getAlias();
+ }
+
public String getTable() {
if (column.getTable() == null) {
return null;
@@ -152,17 +159,6 @@ public class TblColRef implements Serializable {
return InnerDataTypeEnum.DERIVED.getDataType().equals(getDatatype());
}
- /**
- *
- * @param tableName full name : db.table
- * @param columnName columnname
- * @return
- */
- public boolean isSameAs(String tableName, String columnName) {
- return column.isSameAs(tableName, columnName);
- }
-
- @Override
public int hashCode() {
// NOTE: tableRef MUST NOT participate in hashCode().
// Because fixUnknownModel() can change tableRef while TblColRef is held as set/map keys.
@@ -187,6 +183,8 @@ public class TblColRef implements Serializable {
return false;
if (!StringUtils.equals(column.getName(), other.column.getName()))
return false;
+// if ((table == null ? other.table == null : table.equals(other.table)) == false)
+// return false;
return true;
}
@@ -201,4 +199,5 @@ public class TblColRef implements Serializable {
return alias + ":" + tableIdentity + "." + column.getName();
}
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 1f883bd..15fa8ce 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -233,7 +233,7 @@ class ProjectL2Cache {
MetadataManager metaMgr = mgr.getMetadataManager();
- List<TblColRef> allColumns = realization.getAllColumns();
+ Set<TblColRef> allColumns = realization.getAllColumns();
if (allColumns == null || allColumns.isEmpty()) {
logger.error("Realization '" + realization.getCanonicalName() + "' does not report any columns");
return false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index 040cdc5..343ec99 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -1,61 +1,62 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.realization;
-
-import java.util.List;
-
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.IStorageAware;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-public interface IRealization extends IStorageAware {
-
- /**
- * Given the features of a query, check how capable the realization is to answer the query.
- */
- public CapabilityResult isCapable(SQLDigest digest);
-
- /**
- * Get whether this specific realization is a cube or InvertedIndex
- */
- public RealizationType getType();
-
- public DataModelDesc getDataModelDesc();
-
- public String getFactTable();
-
- public List<TblColRef> getAllColumns();
-
- public List<TblColRef> getAllDimensions();
-
- public List<MeasureDesc> getMeasures();
-
- public boolean isReady();
-
- public String getName();
-
- public String getCanonicalName();
-
- public long getDateRangeStart();
-
- public long getDateRangeEnd();
-
- public boolean supportsLimitPushDown();
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.realization;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.IStorageAware;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public interface IRealization extends IStorageAware {
+
+ /**
+ * Given the features of a query, check how capable the realization is to answer the query.
+ */
+ public CapabilityResult isCapable(SQLDigest digest);
+
+ /**
+ * Get whether this specific realization is a cube or InvertedIndex
+ */
+ public RealizationType getType();
+
+ public DataModelDesc getDataModelDesc();
+
+ public String getFactTable();
+
+ public Set<TblColRef> getAllColumns();
+
+ public List<TblColRef> getAllDimensions();
+
+ public List<MeasureDesc> getMeasures();
+
+ public boolean isReady();
+
+ public String getName();
+
+ public String getCanonicalName();
+
+ public long getDateRangeStart();
+
+ public long getDateRangeEnd();
+
+ public boolean supportsLimitPushDown();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 9b3a0fc..57d14d5 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -66,7 +67,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
private volatile IRealization[] realizations = null;
private List<TblColRef> allDimensions = null;
- private List<TblColRef> allColumns = null;
+ private Set<TblColRef> allColumns = null;
private List<MeasureDesc> allMeasures = null;
private long dateRangeStart;
private long dateRangeEnd;
@@ -136,7 +137,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
}
allDimensions = Lists.newArrayList(dimensions);
- allColumns = Lists.newArrayList(columns);
+ allColumns = columns;
allMeasures = Lists.newArrayList(measures);
Collections.sort(realizationList, new Comparator<IRealization>() {
@@ -203,7 +204,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
}
@Override
- public List<TblColRef> getAllColumns() {
+ public Set<TblColRef> getAllColumns() {
init();
return allColumns;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index f4667af..496ac4e 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -65,7 +65,14 @@ public class ITCombinationTest extends ITKylinQueryTest {
*/
@Parameterized.Parameters
public static Collection<Object[]> configs() {
- return Arrays.asList(new Object[][] { { "inner", "on", "v2", false }, { "left", "on", "v1", false }, { "left", "on", "v2", false }, { "inner", "on", "v2", true }, { "left", "on", "v2", true } });
+ return Arrays.asList(new Object[][] { //
+ { "inner", "on", "v2", true }, //
+ { "left", "on", "v1", true }, //
+ { "left", "on", "v2", true }, //
+ //{ "inner", "on", "v2", false }, // exclude view to simply model/cube selection
+ //{ "left", "on", "v1", false }, // exclude view to simply model/cube selection
+ //{ "left", "on", "v2", false }, // exclude view to simply model/cube selection
+ });
}
public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine, boolean excludeViewCubes) throws Exception {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index aa70dbc..8b5ad78 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -84,6 +84,7 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
private final OLAPTable olapTable;
private final String tableName;
private final int[] fields;
+ private String alias;
private ColumnRowType columnRowType;
private OLAPContext context;
@@ -222,9 +223,13 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
}
}
+ public String getAlias() {
+ return alias;
+ }
+
private ColumnRowType buildColumnRowType() {
- String tmpAlias = Integer.toHexString(System.identityHashCode(this));
- TableRef tableRef = TblColRef.tableForUnknownModel(tmpAlias, olapTable.getSourceTable());
+ this.alias = Integer.toHexString(System.identityHashCode(this));
+ TableRef tableRef = TblColRef.tableForUnknownModel(this.alias, olapTable.getSourceTable());
List<TblColRef> columns = new ArrayList<TblColRef>();
for (ColumnDesc sourceColumn : olapTable.getExposedColumns()) {
@@ -239,11 +244,12 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
return tableRef.makeFakeColumn(name);
}
- public void fixColumnRowTypeWithModel(DataModelDesc model) {
- TableRef tableRef = model.findFirstTable(olapTable.getTableName());
+ public void fixColumnRowTypeWithModel(DataModelDesc model, Map<String, String> aliasMap) {
+ String newAlias = aliasMap.get(this.alias);
for (TblColRef col : columnRowType.getAllColumns()) {
- TblColRef.fixUnknownModel(model, tableRef.getAlias(), col);
+ TblColRef.fixUnknownModel(model, newAlias, col);
}
+ this.alias = newAlias;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index aa01086..24fc430 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -45,9 +45,9 @@ import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.query.routing.ModelChooser;
import org.apache.kylin.query.routing.NoRealizationFoundException;
import org.apache.kylin.query.routing.QueryRouter;
import org.apache.kylin.query.schema.OLAPTable;
@@ -89,10 +89,10 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
continue;
}
- IRealization realization = QueryRouter.selectRealization(context);
+ Set<IRealization> candidates = ModelChooser.selectModel(context);
+ IRealization realization = QueryRouter.selectRealization(context, candidates);
context.realization = realization;
- fixModel(context);
doAccessControl(context);
}
} catch (NoRealizationFoundException e) {
@@ -123,13 +123,6 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
return impl.visitChild(this, 0, inputAsEnum, pref);
}
- private void fixModel(OLAPContext context) {
- DataModelDesc model = context.realization.getDataModelDesc();
- for (OLAPTableScan tableScan : context.allTableScans) {
- tableScan.fixColumnRowTypeWithModel(model);
- }
- }
-
private void doAccessControl(OLAPContext context) {
String controllerCls = KylinConfig.getInstanceFromEnv().getQueryAccessController();
if (null != controllerCls && !controllerCls.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
new file mode 100644
index 0000000..4667f4a
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.routing;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.LookupDesc;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.query.relnode.OLAPTableScan;
+import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class ModelChooser {
+
+ public static Set<IRealization> selectModel(OLAPContext context) {
+ Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(context);
+ OLAPTableScan firstTable = context.firstTableScan;
+ List<JoinDesc> joins = context.joins;
+
+ for (DataModelDesc model : modelMap.keySet()) {
+ Map<String, String> aliasMap = matches(model, firstTable, joins);
+ if (aliasMap != null) {
+ fixModel(context, model, aliasMap);
+ return modelMap.get(model);
+ }
+ }
+
+ throw new NoRealizationFoundException("No model found by first table " + firstTable.getOlapTable().getTableName() + " and joins " + joins);
+ }
+
+ private static Map<String, String> matches(DataModelDesc model, OLAPTableScan firstTable, List<JoinDesc> joins) {
+ Map<String, String> result = Maps.newHashMap();
+
+ // no join special case
+ if (joins.isEmpty()) {
+ TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName());
+ if (tableRef == null)
+ return null;
+ result.put(firstTable.getAlias(), tableRef.getAlias());
+ return result;
+ }
+
+ // the greedy match is not perfect but works for the moment
+ Map<String, List<JoinDesc>> modelJoinsMap = model.getJoinsMap();
+ for (JoinDesc queryJoin : joins) {
+ String fkTable = queryJoin.getForeignKeyColumns()[0].getTable();
+ List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable);
+ if (modelJoins == null)
+ return null;
+
+ JoinDesc matchJoin = null;
+ for (JoinDesc modelJoin : modelJoins) {
+ if (modelJoin.matches(queryJoin)) {
+ matchJoin = modelJoin;
+ break;
+ }
+ }
+ if (matchJoin == null)
+ return null;
+
+ matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result);
+ matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result);
+ }
+ return result;
+ }
+
+ private static void matchesAdd(String origAlias, String targetAlias, Map<String, String> result) {
+ String existingTarget = result.put(origAlias, targetAlias);
+ Preconditions.checkState(existingTarget == null || existingTarget.equals(targetAlias));
+ }
+
+ private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(OLAPContext context) {
+ KylinConfig kylinConfig = context.olapSchema.getConfig();
+ String projectName = context.olapSchema.getProjectName();
+ String factTableName = context.firstTableScan.getOlapTable().getTableName();
+ Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, factTableName);
+
+ final Map<DataModelDesc, Set<IRealization>> models = Maps.newHashMap();
+ final Map<DataModelDesc, RealizationCost> costs = Maps.newHashMap();
+ for (IRealization real : realizations) {
+ if (real.isReady() == false)
+ continue;
+ if (real.getAllColumns().containsAll(context.allColumns) == false)
+ continue;
+ if (RemoveBlackoutRealizationsRule.accept(real) == false)
+ continue;
+
+ RealizationCost cost = new RealizationCost(real);
+ DataModelDesc m = real.getDataModelDesc();
+ Set<IRealization> set = models.get(m);
+ if (set == null) {
+ set = Sets.newHashSet();
+ set.add(real);
+ models.put(m, set);
+ costs.put(m, cost);
+ } else {
+ set.add(real);
+ RealizationCost curCost = costs.get(m);
+ if (cost.compareTo(curCost) < 0)
+ costs.put(m, cost);
+ }
+ }
+
+ // order model by cheapest realization cost
+ TreeMap<DataModelDesc, Set<IRealization>> result = Maps.newTreeMap(new Comparator<DataModelDesc>() {
+ @Override
+ public int compare(DataModelDesc o1, DataModelDesc o2) {
+ return costs.get(o1).compareTo(costs.get(o2));
+ }
+ });
+ result.putAll(models);
+
+ return result;
+ }
+
+ private static void fixModel(OLAPContext context, DataModelDesc model, Map<String, String> aliasMap) {
+ for (OLAPTableScan tableScan : context.allTableScans) {
+ tableScan.fixColumnRowTypeWithModel(model, aliasMap);
+ }
+ }
+
+ private static class RealizationCost implements Comparable<RealizationCost> {
+ final public int priority;
+ final public int cost;
+
+ public RealizationCost(IRealization real) {
+ // ref Candidate.PRIORITIES
+ this.priority = Candidate.PRIORITIES.get(real.getType());
+
+ // ref CubeInstance.getCost()
+ int c = real.getAllDimensions().size() * CubeInstance.COST_WEIGHT_DIMENSION + real.getMeasures().size() * CubeInstance.COST_WEIGHT_MEASURE;
+ for (LookupDesc lookup : real.getDataModelDesc().getLookups()) {
+ if (lookup.getJoin().isInnerJoin())
+ c += CubeInstance.COST_WEIGHT_INNER_JOIN;
+ }
+ this.cost = c;
+ }
+
+ @Override
+ public int compareTo(RealizationCost o) {
+ int comp = this.priority - o.priority;
+ if (comp != 0)
+ return comp;
+ else
+ return this.cost - o.cost;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
index 69ebfa6..2975cf7 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
@@ -23,7 +23,6 @@ import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.CapabilityResult.DimensionAsMeasure;
@@ -42,13 +41,10 @@ public class QueryRouter {
private static final Logger logger = LoggerFactory.getLogger(QueryRouter.class);
- public static IRealization selectRealization(OLAPContext olapContext) throws NoRealizationFoundException {
+ public static IRealization selectRealization(OLAPContext olapContext, Set<IRealization> realizations) throws NoRealizationFoundException {
- ProjectManager prjMgr = ProjectManager.getInstance(olapContext.olapSchema.getConfig());
- logger.info("The project manager's reference is " + prjMgr);
String factTableName = olapContext.firstTableScan.getTableName();
String projectName = olapContext.olapSchema.getProjectName();
- Set<IRealization> realizations = prjMgr.getRealizationsByTable(projectName, factTableName);
SQLDigest sqlDigest = olapContext.getSQLDigest();
List<Candidate> candidates = Lists.newArrayListWithCapacity(realizations.size());
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
index f299d17..f309757 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
@@ -22,6 +22,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.routing.RoutingRule;
@@ -34,19 +35,22 @@ public class RemoveBlackoutRealizationsRule extends RoutingRule {
public static Set<String> blackList = Sets.newHashSet();
public static Set<String> whiteList = Sets.newHashSet();
+ public static boolean accept(IRealization real) {
+ if (blackList.contains(real.getCanonicalName()))
+ return false;
+ if (!whiteList.isEmpty() && !whiteList.contains(real.getCanonicalName()))
+ return false;
+
+ return true;
+ }
+
@Override
public void apply(List<Candidate> candidates) {
for (Iterator<Candidate> iterator = candidates.iterator(); iterator.hasNext();) {
Candidate candidate = iterator.next();
- if (blackList.contains(candidate.getRealization().getCanonicalName())) {
- iterator.remove();
- continue;
- }
-
- if (!whiteList.isEmpty() && !whiteList.contains(candidate.getRealization().getCanonicalName())) {
+ if (!accept(candidate.getRealization())) {
iterator.remove();
- continue;
}
}
}