You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/10/26 10:24:36 UTC
[1/6] kylin git commit: KYLIN-2106 integer encoding bug fix [Forced
Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-1971 0f8b2393d -> c1e6ff680 (forced update)
KYLIN-2106 integer encoding bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85717fe3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85717fe3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85717fe3
Branch: refs/heads/KYLIN-1971
Commit: 85717fe33712fe1b0f6d78e371aa9194ce5f1d4a
Parents: eb39437
Author: Jason <ji...@163.com>
Authored: Tue Oct 25 18:39:51 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Tue Oct 25 20:19:55 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 12 ++++++++++--
webapp/app/js/controllers/cubeMeasures.js | 16 ++++++++++++----
2 files changed, 22 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/85717fe3/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index be90f65..0ba321c 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -27,18 +27,24 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
angular.forEach($scope.cubeMetaFrame.rowkey.rowkey_columns,function(item){
//var _isDictionaries = item.encoding === "dict"?"true":"false";
var _isFixedLength = item.encoding.substring(0,12) === "fixed_length"?"true":"false";//fixed_length:12
- var _isIntLength = item.encoding.substring(0,3) === "int"?"true":"false";//fixed_length:12
+ var _isIntegerLength = item.encoding.substring(0,7) === "integer"?"true":"false";
+ var _isIntLength = item.encoding.substring(0,3) === "int"?"true":"false";
var _encoding = item.encoding;
var _valueLength ;
if(_isFixedLength !=="false"){
_valueLength = item.encoding.substring(13,item.encoding.length);
_encoding = "fixed_length";
}
- if(_isIntLength!="false"){
+ if(_isIntLength!="false" && _isIntegerLength=="false" ){
_valueLength = item.encoding.substring(4,item.encoding.length);
_encoding = "int";
}
+ if(_isIntegerLength!="false" ){
+ _valueLength = item.encoding.substring(8,item.encoding.length);
+ _encoding = "integer";
+ }
+
var rowkeyObj = {
column:item.column,
encoding:_encoding,
@@ -66,6 +72,8 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
}
else if(item.encoding=="int" && item.valueLength){
encoding = "int:"+item.valueLength;
+ }else if(item.encoding=="integer" && item.valueLength){
+ encoding = "integer:"+item.valueLength;
}else{
encoding = item.encoding;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/85717fe3/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 006c6c2..dcf26b7 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -54,17 +54,24 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
var _name=configuration.slice(14);
var item=$scope.newMeasure.function.configuration[configuration];
var _isFixedLength = item.substring(0,12) === "fixed_length"?"true":"false";//fixed_length:12
- var _isIntLength = item.substring(0,3) === "int"?"true":"false";//fixed_length:12
+ var _isIntegerLength = item.substring(0,7) === "integer"?"true":"false";
+ var _isIntLength = item.substring(0,3) === "int"?"true":"false";
var _encoding = item;
var _valueLength = 0 ;
if(_isFixedLength !=="false"){
_valueLength = item.substring(13,item.length);
_encoding = "fixed_length";
}
- if(_isIntLength!="false"){
+ if(_isIntLength!="false" && _isIntegerLength=="false" ){
_valueLength = item.substring(4,item.length);
_encoding = "int";
}
+
+ if(_isIntegerLength!="false" ){
+ _valueLength = item.substring(8,item.length);
+ _encoding = "integer";
+ }
+
$scope.GroupBy = {
name:_name,
encoding:_encoding,
@@ -172,8 +179,9 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
if(item.encoding!=="dict" && item.encoding!=="date"&& item.encoding!=="time"){
if(item.encoding=="fixed_length" && item.valueLength){
encoding = "fixed_length:"+item.valueLength;
- }
- else if(item.encoding=="int" && item.valueLength){
+ }else if(item.encoding=="integer" && item.valueLength){
+ encoding = "integer:"+item.valueLength;
+ }else if(item.encoding=="int" && item.valueLength){
encoding = "int:"+item.valueLength;
}else{
encoding = item.encoding;
[4/6] kylin git commit: KYLIN-1321 tips update
Posted by li...@apache.org.
KYLIN-1321 tips update
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/151ec3bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/151ec3bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/151ec3bd
Branch: refs/heads/KYLIN-1971
Commit: 151ec3bd13c290ef250df10cef36ee94f77abaaa
Parents: 6350e31
Author: Jason <ji...@163.com>
Authored: Tue Oct 25 21:53:07 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Tue Oct 25 21:53:07 2016 +0800
----------------------------------------------------------------------
webapp/app/partials/cubeDesigner/dimensions.html | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/151ec3bd/webapp/app/partials/cubeDesigner/dimensions.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/dimensions.html b/webapp/app/partials/cubeDesigner/dimensions.html
index 0c4236e..d826575 100644
--- a/webapp/app/partials/cubeDesigner/dimensions.html
+++ b/webapp/app/partials/cubeDesigner/dimensions.html
@@ -224,7 +224,7 @@
<script type="text/ng-template" id="autoGenDimension.html">
<div class="modal-header large-popover">
- <h4 class="box-title lighter">Auto Generate Dimensions <small>This is a helper for you to batch generate dimensions.</small><i kylinpopover placement="right" title="Auto Generate Rules" template="AutoDimensionsTip.html" class="fa fa-info-circle"></i></h4>
+ <h4 class="box-title lighter">Auto Generate Dimensions <i kylinpopover placement="right" title="Auto Generate Rules" template="AutoDimensionsTip.html" class="fa fa-info-circle"></i> <small>This is a helper for you to batch generate dimensions.</small></h4>
</div>
<div class="modal-body">
<div class="row">
@@ -310,7 +310,7 @@
<ol class="text-info">
<li>Column is disabled once there is already a dimension referred this column.</li>
<li>In fact table: one normal dimension will be auto generated per column.</li>
- <li>In lookup table: one derived dimension will be auto generated for all the selected columns.</li>
+ <li>In lookup table: you can choose to generate a derived column or a normal column.</li>
</ol>
</div>
</div>
[5/6] kylin git commit: KYLIN-1971 ModelChooser,
match Model against query joins
Posted by li...@apache.org.
KYLIN-1971 ModelChooser, match Model against query joins
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5cdd2252
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5cdd2252
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5cdd2252
Branch: refs/heads/KYLIN-1971
Commit: 5cdd22523a68faa4879092d4b2510ea461609f6e
Parents: 151ec3b
Author: Yang Li <li...@apache.org>
Authored: Tue Oct 25 06:16:15 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Oct 26 11:06:42 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 13 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 8 +-
.../kylin/metadata/model/DataModelDesc.java | 13 +-
.../kylin/metadata/model/FunctionDesc.java | 18 --
.../apache/kylin/metadata/model/JoinDesc.java | 44 ++++-
.../apache/kylin/metadata/model/TableRef.java | 4 +-
.../apache/kylin/metadata/model/TblColRef.java | 21 ++-
.../kylin/metadata/project/ProjectL2Cache.java | 2 +-
.../metadata/realization/IRealization.java | 123 ++++++-------
.../kylin/storage/hybrid/HybridInstance.java | 7 +-
.../apache/kylin/query/ITCombinationTest.java | 9 +-
.../kylin/query/relnode/OLAPTableScan.java | 16 +-
.../relnode/OLAPToEnumerableConverter.java | 13 +-
.../kylin/query/routing/ModelChooser.java | 178 +++++++++++++++++++
.../apache/kylin/query/routing/QueryRouter.java | 6 +-
.../rules/RemoveBlackoutRealizationsRule.java | 18 +-
16 files changed, 351 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 720690d..2ccdde7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -21,6 +21,7 @@ package org.apache.kylin.cube;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
+import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
@@ -52,9 +53,9 @@ import com.google.common.collect.Lists;
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class CubeInstance extends RootPersistentEntity implements IRealization, IBuildable {
- private static final int COST_WEIGHT_MEASURE = 1;
- private static final int COST_WEIGHT_DIMENSION = 10;
- private static final int COST_WEIGHT_INNER_JOIN = 100;
+ public static final int COST_WEIGHT_MEASURE = 1;
+ public static final int COST_WEIGHT_DIMENSION = 10;
+ public static final int COST_WEIGHT_INNER_JOIN = 100;
public static CubeInstance create(String cubeName, CubeDesc cubeDesc) {
CubeInstance cubeInstance = new CubeInstance();
@@ -374,7 +375,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
for (LookupDesc lookupDesc : this.getDescriptor().getModel().getLookups()) {
// more tables, more cost
- if ("inner".equals(lookupDesc.getJoin().getType())) {
+ if (lookupDesc.getJoin().isInnerJoin()) {
// inner join cost is bigger than left join, as it will filter some records
calculatedCost += COST_WEIGHT_INNER_JOIN;
}
@@ -389,8 +390,8 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
@Override
- public List<TblColRef> getAllColumns() {
- return Lists.newArrayList(getDescriptor().listAllColumns());
+ public Set<TblColRef> getAllColumns() {
+ return getDescriptor().listAllColumns();
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 772297f..3bf0de9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -20,12 +20,11 @@ package org.apache.kylin.metadata.model;
import java.io.Serializable;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import org.apache.commons.lang.StringUtils;
import org.apache.kylin.metadata.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
@@ -154,11 +153,6 @@ public class ColumnDesc implements Serializable {
}
}
- public boolean isSameAs(String tableName, String columnName) {
- return StringUtils.equalsIgnoreCase(table.getIdentity(), tableName) && //
- StringUtils.equalsIgnoreCase(name, columnName);
- }
-
@Override
public int hashCode() {
final int prime = 31;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index a7734f1..7c39a25 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -86,6 +86,7 @@ public class DataModelDesc extends RootPersistentEntity {
private List<TableRef> lookupTableRefs = Lists.newArrayList();
private Map<String, TableRef> aliasMap = Maps.newHashMap(); // a table has exactly one alias
private Map<String, TableRef> tableNameMap = Maps.newHashMap(); // a table maybe referenced by multiple names
+ private Map<String, List<JoinDesc>> joinsMap = Maps.newHashMap();
/**
* Error messages during resolving json metadata
@@ -274,6 +275,7 @@ public class DataModelDesc extends RootPersistentEntity {
lookupTableRefs.clear();
aliasMap.clear();
tableNameMap.clear();
+ joinsMap.clear();
initTableAlias(tables);
initJoinColumns();
@@ -378,10 +380,19 @@ public class DataModelDesc extends RootPersistentEntity {
}
}
+ List<JoinDesc> list = joinsMap.get(factTableRef.getTableIdentity());
+ if (list == null)
+ joinsMap.put(factTableRef.getTableIdentity(), list = Lists.newArrayListWithCapacity(4));
+ list.add(join);
}
}
+
+ public Map<String, List<JoinDesc>> getJoinsMap() {
+ return joinsMap;
+ }
- /** * Add error info and thrown exception out
+ /**
+ * Add error info and thrown exception out
*
* @param message
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index a2c5756..e7d5186 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -19,7 +19,6 @@
package org.apache.kylin.metadata.model;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Set;
@@ -238,23 +237,6 @@ public class FunctionDesc {
this.returnDataType = DataType.getType(returnType);
}
- public TblColRef selectTblColRef(Collection<TblColRef> metricColumns, String factTableName) {
- if (this.isCount())
- return null; // count is not about any column but the whole row
-
- ParameterDesc parameter = this.getParameter();
- if (parameter == null)
- return null;
-
- String columnName = parameter.getValue();
- for (TblColRef col : metricColumns) {
- if (col.isSameAs(factTableName, columnName)) {
- return col;
- }
- }
- return null;
- }
-
public HashMap<String, String> getConfiguration() {
return configuration;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index 458367d..04fbf62 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -50,10 +50,18 @@ public class JoinDesc {
foreignKeyColumns = tt;
}
+ public boolean isInnerJoin() {
+ return "INNER".equalsIgnoreCase(type);
+ }
+
+ public boolean isLeftJoin() {
+ return "LEFT".equalsIgnoreCase(type);
+ }
+
public String getType() {
return type;
}
-
+
public void setType(String type) {
this.type = type;
}
@@ -110,9 +118,9 @@ public class JoinDesc {
return false;
JoinDesc other = (JoinDesc) obj;
- if (!this.columnsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns))
+ if (!this.colRefsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns))
return false;
- if (!this.columnsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns))
+ if (!this.colRefsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns))
return false;
if (!this.type.equalsIgnoreCase(other.getType()))
@@ -120,13 +128,41 @@ public class JoinDesc {
return true;
}
- private boolean columnsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) {
+ private boolean colRefsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) {
if (a.length != b.length)
return false;
return Arrays.asList(a).containsAll(Arrays.asList(b));
}
+ // equals() without alias
+ public boolean matches(JoinDesc other) {
+ if (!this.type.equalsIgnoreCase(other.getType()))
+ return false;
+ if (!this.columnsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns))
+ return false;
+ if (!this.columnsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns))
+ return false;
+
+ return true;
+ }
+
+ private boolean columnsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) {
+ if (a.length != b.length)
+ return false;
+
+ int match = 0;
+ for (int i = 0; i < a.length; i++) {
+ for (int j = 0; j < b.length; j++) {
+ if (a[i].equals(b[j])) {
+ match++;
+ break;
+ }
+ }
+ }
+ return match == a.length;
+ }
+
@Override
public String toString() {
return "JoinDesc [type=" + type + ", primary_key=" + Arrays.toString(primaryKey) + ", foreign_key=" + Arrays.toString(foreignKey) + "]";
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
index 186ff35..254ce18 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
@@ -86,9 +86,9 @@ public class TableRef {
TableRef t = (TableRef) o;
- if (!model.equals(t.model))
+ if ((model == null ? t.model == null : model.getName().equals(t.model.getName())) == false)
return false;
- if (!alias.equals(t.alias))
+ if ((alias == null ? t.alias == null : alias.equals(t.alias)) == false)
return false;
if (!table.getIdentity().equals(t.table.getIdentity()))
return false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 69569dd..8841ee5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -64,6 +64,9 @@ public class TblColRef implements Serializable {
}
private static final DataModelDesc UNKNOWN_MODEL = new DataModelDesc();
+ static {
+ UNKNOWN_MODEL.setName("UNKNOWN_MODEL");
+ }
public static TableRef tableForUnknownModel(String tempTableAlias, TableDesc table) {
return new TableRef(UNKNOWN_MODEL, tempTableAlias, table);
@@ -119,6 +122,10 @@ public class TblColRef implements Serializable {
return table;
}
+ public String getTableAlias() {
+ return table.getAlias();
+ }
+
public String getTable() {
if (column.getTable() == null) {
return null;
@@ -152,17 +159,6 @@ public class TblColRef implements Serializable {
return InnerDataTypeEnum.DERIVED.getDataType().equals(getDatatype());
}
- /**
- *
- * @param tableName full name : db.table
- * @param columnName columnname
- * @return
- */
- public boolean isSameAs(String tableName, String columnName) {
- return column.isSameAs(tableName, columnName);
- }
-
- @Override
public int hashCode() {
// NOTE: tableRef MUST NOT participate in hashCode().
// Because fixUnknownModel() can change tableRef while TblColRef is held as set/map keys.
@@ -187,6 +183,8 @@ public class TblColRef implements Serializable {
return false;
if (!StringUtils.equals(column.getName(), other.column.getName()))
return false;
+// if ((table == null ? other.table == null : table.equals(other.table)) == false)
+// return false;
return true;
}
@@ -201,4 +199,5 @@ public class TblColRef implements Serializable {
return alias + ":" + tableIdentity + "." + column.getName();
}
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 1f883bd..15fa8ce 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -233,7 +233,7 @@ class ProjectL2Cache {
MetadataManager metaMgr = mgr.getMetadataManager();
- List<TblColRef> allColumns = realization.getAllColumns();
+ Set<TblColRef> allColumns = realization.getAllColumns();
if (allColumns == null || allColumns.isEmpty()) {
logger.error("Realization '" + realization.getCanonicalName() + "' does not report any columns");
return false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index 040cdc5..343ec99 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -1,61 +1,62 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.realization;
-
-import java.util.List;
-
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.IStorageAware;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-public interface IRealization extends IStorageAware {
-
- /**
- * Given the features of a query, check how capable the realization is to answer the query.
- */
- public CapabilityResult isCapable(SQLDigest digest);
-
- /**
- * Get whether this specific realization is a cube or InvertedIndex
- */
- public RealizationType getType();
-
- public DataModelDesc getDataModelDesc();
-
- public String getFactTable();
-
- public List<TblColRef> getAllColumns();
-
- public List<TblColRef> getAllDimensions();
-
- public List<MeasureDesc> getMeasures();
-
- public boolean isReady();
-
- public String getName();
-
- public String getCanonicalName();
-
- public long getDateRangeStart();
-
- public long getDateRangeEnd();
-
- public boolean supportsLimitPushDown();
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.realization;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.IStorageAware;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public interface IRealization extends IStorageAware {
+
+ /**
+ * Given the features of a query, check how capable the realization is to answer the query.
+ */
+ public CapabilityResult isCapable(SQLDigest digest);
+
+ /**
+ * Get whether this specific realization is a cube or InvertedIndex
+ */
+ public RealizationType getType();
+
+ public DataModelDesc getDataModelDesc();
+
+ public String getFactTable();
+
+ public Set<TblColRef> getAllColumns();
+
+ public List<TblColRef> getAllDimensions();
+
+ public List<MeasureDesc> getMeasures();
+
+ public boolean isReady();
+
+ public String getName();
+
+ public String getCanonicalName();
+
+ public long getDateRangeStart();
+
+ public long getDateRangeEnd();
+
+ public boolean supportsLimitPushDown();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 9b3a0fc..57d14d5 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -66,7 +67,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
private volatile IRealization[] realizations = null;
private List<TblColRef> allDimensions = null;
- private List<TblColRef> allColumns = null;
+ private Set<TblColRef> allColumns = null;
private List<MeasureDesc> allMeasures = null;
private long dateRangeStart;
private long dateRangeEnd;
@@ -136,7 +137,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
}
allDimensions = Lists.newArrayList(dimensions);
- allColumns = Lists.newArrayList(columns);
+ allColumns = columns;
allMeasures = Lists.newArrayList(measures);
Collections.sort(realizationList, new Comparator<IRealization>() {
@@ -203,7 +204,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
}
@Override
- public List<TblColRef> getAllColumns() {
+ public Set<TblColRef> getAllColumns() {
init();
return allColumns;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index f4667af..496ac4e 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -65,7 +65,14 @@ public class ITCombinationTest extends ITKylinQueryTest {
*/
@Parameterized.Parameters
public static Collection<Object[]> configs() {
- return Arrays.asList(new Object[][] { { "inner", "on", "v2", false }, { "left", "on", "v1", false }, { "left", "on", "v2", false }, { "inner", "on", "v2", true }, { "left", "on", "v2", true } });
+ return Arrays.asList(new Object[][] { //
+ { "inner", "on", "v2", true }, //
+ { "left", "on", "v1", true }, //
+ { "left", "on", "v2", true }, //
+ //{ "inner", "on", "v2", false }, // exclude view to simply model/cube selection
+ //{ "left", "on", "v1", false }, // exclude view to simply model/cube selection
+ //{ "left", "on", "v2", false }, // exclude view to simply model/cube selection
+ });
}
public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine, boolean excludeViewCubes) throws Exception {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index aa70dbc..8b5ad78 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -84,6 +84,7 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
private final OLAPTable olapTable;
private final String tableName;
private final int[] fields;
+ private String alias;
private ColumnRowType columnRowType;
private OLAPContext context;
@@ -222,9 +223,13 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
}
}
+ public String getAlias() {
+ return alias;
+ }
+
private ColumnRowType buildColumnRowType() {
- String tmpAlias = Integer.toHexString(System.identityHashCode(this));
- TableRef tableRef = TblColRef.tableForUnknownModel(tmpAlias, olapTable.getSourceTable());
+ this.alias = Integer.toHexString(System.identityHashCode(this));
+ TableRef tableRef = TblColRef.tableForUnknownModel(this.alias, olapTable.getSourceTable());
List<TblColRef> columns = new ArrayList<TblColRef>();
for (ColumnDesc sourceColumn : olapTable.getExposedColumns()) {
@@ -239,11 +244,12 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
return tableRef.makeFakeColumn(name);
}
- public void fixColumnRowTypeWithModel(DataModelDesc model) {
- TableRef tableRef = model.findFirstTable(olapTable.getTableName());
+ public void fixColumnRowTypeWithModel(DataModelDesc model, Map<String, String> aliasMap) {
+ String newAlias = aliasMap.get(this.alias);
for (TblColRef col : columnRowType.getAllColumns()) {
- TblColRef.fixUnknownModel(model, tableRef.getAlias(), col);
+ TblColRef.fixUnknownModel(model, newAlias, col);
}
+ this.alias = newAlias;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index aa01086..24fc430 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -45,9 +45,9 @@ import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.query.routing.ModelChooser;
import org.apache.kylin.query.routing.NoRealizationFoundException;
import org.apache.kylin.query.routing.QueryRouter;
import org.apache.kylin.query.schema.OLAPTable;
@@ -89,10 +89,10 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
continue;
}
- IRealization realization = QueryRouter.selectRealization(context);
+ Set<IRealization> candidates = ModelChooser.selectModel(context);
+ IRealization realization = QueryRouter.selectRealization(context, candidates);
context.realization = realization;
- fixModel(context);
doAccessControl(context);
}
} catch (NoRealizationFoundException e) {
@@ -123,13 +123,6 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
return impl.visitChild(this, 0, inputAsEnum, pref);
}
- private void fixModel(OLAPContext context) {
- DataModelDesc model = context.realization.getDataModelDesc();
- for (OLAPTableScan tableScan : context.allTableScans) {
- tableScan.fixColumnRowTypeWithModel(model);
- }
- }
-
private void doAccessControl(OLAPContext context) {
String controllerCls = KylinConfig.getInstanceFromEnv().getQueryAccessController();
if (null != controllerCls && !controllerCls.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
new file mode 100644
index 0000000..4667f4a
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.routing;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.LookupDesc;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.query.relnode.OLAPTableScan;
+import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class ModelChooser {
+
+ public static Set<IRealization> selectModel(OLAPContext context) {
+ Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(context);
+ OLAPTableScan firstTable = context.firstTableScan;
+ List<JoinDesc> joins = context.joins;
+
+ for (DataModelDesc model : modelMap.keySet()) {
+ Map<String, String> aliasMap = matches(model, firstTable, joins);
+ if (aliasMap != null) {
+ fixModel(context, model, aliasMap);
+ return modelMap.get(model);
+ }
+ }
+
+ throw new NoRealizationFoundException("No model found by first table " + firstTable.getOlapTable().getTableName() + " and joins " + joins);
+ }
+
+ private static Map<String, String> matches(DataModelDesc model, OLAPTableScan firstTable, List<JoinDesc> joins) {
+ Map<String, String> result = Maps.newHashMap();
+
+ // no join special case
+ if (joins.isEmpty()) {
+ TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName());
+ if (tableRef == null)
+ return null;
+ result.put(firstTable.getAlias(), tableRef.getAlias());
+ return result;
+ }
+
+ // the greedy match is not perfect but works for the moment
+ Map<String, List<JoinDesc>> modelJoinsMap = model.getJoinsMap();
+ for (JoinDesc queryJoin : joins) {
+ String fkTable = queryJoin.getForeignKeyColumns()[0].getTable();
+ List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable);
+ if (modelJoins == null)
+ return null;
+
+ JoinDesc matchJoin = null;
+ for (JoinDesc modelJoin : modelJoins) {
+ if (modelJoin.matches(queryJoin)) {
+ matchJoin = modelJoin;
+ break;
+ }
+ }
+ if (matchJoin == null)
+ return null;
+
+ matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result);
+ matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result);
+ }
+ return result;
+ }
+
+ private static void matchesAdd(String origAlias, String targetAlias, Map<String, String> result) {
+ String existingTarget = result.put(origAlias, targetAlias);
+ Preconditions.checkState(existingTarget == null || existingTarget.equals(targetAlias));
+ }
+
+ private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(OLAPContext context) {
+ KylinConfig kylinConfig = context.olapSchema.getConfig();
+ String projectName = context.olapSchema.getProjectName();
+ String factTableName = context.firstTableScan.getOlapTable().getTableName();
+ Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, factTableName);
+
+ final Map<DataModelDesc, Set<IRealization>> models = Maps.newHashMap();
+ final Map<DataModelDesc, RealizationCost> costs = Maps.newHashMap();
+ for (IRealization real : realizations) {
+ if (real.isReady() == false)
+ continue;
+ if (real.getAllColumns().containsAll(context.allColumns) == false)
+ continue;
+ if (RemoveBlackoutRealizationsRule.accept(real) == false)
+ continue;
+
+ RealizationCost cost = new RealizationCost(real);
+ DataModelDesc m = real.getDataModelDesc();
+ Set<IRealization> set = models.get(m);
+ if (set == null) {
+ set = Sets.newHashSet();
+ set.add(real);
+ models.put(m, set);
+ costs.put(m, cost);
+ } else {
+ set.add(real);
+ RealizationCost curCost = costs.get(m);
+ if (cost.compareTo(curCost) < 0)
+ costs.put(m, cost);
+ }
+ }
+
+ // order model by cheapest realization cost
+ TreeMap<DataModelDesc, Set<IRealization>> result = Maps.newTreeMap(new Comparator<DataModelDesc>() {
+ @Override
+ public int compare(DataModelDesc o1, DataModelDesc o2) {
+ return costs.get(o1).compareTo(costs.get(o2));
+ }
+ });
+ result.putAll(models);
+
+ return result;
+ }
+
+ private static void fixModel(OLAPContext context, DataModelDesc model, Map<String, String> aliasMap) {
+ for (OLAPTableScan tableScan : context.allTableScans) {
+ tableScan.fixColumnRowTypeWithModel(model, aliasMap);
+ }
+ }
+
+ private static class RealizationCost implements Comparable<RealizationCost> {
+ final public int priority;
+ final public int cost;
+
+ public RealizationCost(IRealization real) {
+ // ref Candidate.PRIORITIES
+ this.priority = Candidate.PRIORITIES.get(real.getType());
+
+ // ref CubeInstance.getCost()
+ int c = real.getAllDimensions().size() * CubeInstance.COST_WEIGHT_DIMENSION + real.getMeasures().size() * CubeInstance.COST_WEIGHT_MEASURE;
+ for (LookupDesc lookup : real.getDataModelDesc().getLookups()) {
+ if (lookup.getJoin().isInnerJoin())
+ c += CubeInstance.COST_WEIGHT_INNER_JOIN;
+ }
+ this.cost = c;
+ }
+
+ @Override
+ public int compareTo(RealizationCost o) {
+ int comp = this.priority - o.priority;
+ if (comp != 0)
+ return comp;
+ else
+ return this.cost - o.cost;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
index 69ebfa6..2975cf7 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
@@ -23,7 +23,6 @@ import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.CapabilityResult.DimensionAsMeasure;
@@ -42,13 +41,10 @@ public class QueryRouter {
private static final Logger logger = LoggerFactory.getLogger(QueryRouter.class);
- public static IRealization selectRealization(OLAPContext olapContext) throws NoRealizationFoundException {
+ public static IRealization selectRealization(OLAPContext olapContext, Set<IRealization> realizations) throws NoRealizationFoundException {
- ProjectManager prjMgr = ProjectManager.getInstance(olapContext.olapSchema.getConfig());
- logger.info("The project manager's reference is " + prjMgr);
String factTableName = olapContext.firstTableScan.getTableName();
String projectName = olapContext.olapSchema.getProjectName();
- Set<IRealization> realizations = prjMgr.getRealizationsByTable(projectName, factTableName);
SQLDigest sqlDigest = olapContext.getSQLDigest();
List<Candidate> candidates = Lists.newArrayListWithCapacity(realizations.size());
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cdd2252/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
index f299d17..f309757 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
@@ -22,6 +22,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.routing.RoutingRule;
@@ -34,19 +35,22 @@ public class RemoveBlackoutRealizationsRule extends RoutingRule {
public static Set<String> blackList = Sets.newHashSet();
public static Set<String> whiteList = Sets.newHashSet();
+ public static boolean accept(IRealization real) {
+ if (blackList.contains(real.getCanonicalName()))
+ return false;
+ if (!whiteList.isEmpty() && !whiteList.contains(real.getCanonicalName()))
+ return false;
+
+ return true;
+ }
+
@Override
public void apply(List<Candidate> candidates) {
for (Iterator<Candidate> iterator = candidates.iterator(); iterator.hasNext();) {
Candidate candidate = iterator.next();
- if (blackList.contains(candidate.getRealization().getCanonicalName())) {
- iterator.remove();
- continue;
- }
-
- if (!whiteList.isEmpty() && !whiteList.contains(candidate.getRealization().getCanonicalName())) {
+ if (!accept(candidate.getRealization())) {
iterator.remove();
- continue;
}
}
}
[2/6] kylin git commit: KYLIN-2082 straming config edit
Posted by li...@apache.org.
KYLIN-2082 straming config edit
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92c823bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92c823bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92c823bb
Branch: refs/heads/KYLIN-1971
Commit: 92c823bb68c1280ae1ae48d325fa1c678a4c7812
Parents: 85717fe
Author: luguosheng <55...@qq.com>
Authored: Tue Oct 25 18:49:02 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Tue Oct 25 20:20:25 2016 +0800
----------------------------------------------------------------------
.../partials/cubeDesigner/streamingConfig.html | 153 ++++++++-----------
1 file changed, 64 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/92c823bb/webapp/app/partials/cubeDesigner/streamingConfig.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/streamingConfig.html b/webapp/app/partials/cubeDesigner/streamingConfig.html
index 0148ac3..8bdcd25 100644
--- a/webapp/app/partials/cubeDesigner/streamingConfig.html
+++ b/webapp/app/partials/cubeDesigner/streamingConfig.html
@@ -124,176 +124,151 @@
</accordion-group>
</accordion>
-
<hr/>
- <!--Advanced setting-->
<accordion>
- <accordion-group is-open="state.isStreamingAdOpen" ng-init="state.isStreamingAdOpen=true">
+
+ <accordion-group is-open="state.isParserHeaderOpen=true">
<accordion-heading>
- Advanced Setting
+ Parser Setting
<i class="pull-right glyphicon"
- ng-class="{'glyphicon-chevron-down': state.isStreamingAdOpen, 'glyphicon-chevron-right': !state.isStreamingAdOpen}"></i>
+ ng-class="{'glyphicon-chevron-down': state.isParserHeaderOpen, 'glyphicon-chevron-right': !state.isParserHeaderOpen}"></i>
</accordion-heading>
<div class="form-group middle-popover" ng-class="{'required':state.mode=='edit'}">
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right">
- <b>Timeout</b>
- <i class="fa fa-info-circle" kylinpopover placement="right" title="Timeout" template="TimeoutTip.html"></i>
+ <b>Parser Name</b>
+ <i class="fa fa-info-circle" kylinpopover placement="right" title="Parser Name" template="ParserName.html"></i>
</label>
<div class="col-xs-12 col-sm-6"
- ng-class="{'has-error':form.cube_streaming_form.timeout.$invalid && (form.cube_streaming_form.timeout.$dirty||form.cube_streaming_form.$submitted)}">
- <input ng-if="state.mode=='edit'" name="timeout" required ng-model="kafkaMeta.timeout" type="text"
- placeholder="Input kafkaConfig timeout"
- ng-pattern="/^\+?[1-9][0-9]*$/"
+ ng-class="{'has-error':form.cube_streaming_form.parserName.$invalid && (form.cube_streaming_form.parserName.$dirty||form.cube_streaming_form.$submitted)}">
+ <input ng-if="state.mode=='edit'" name="parserName" required ng-model="kafkaMeta.parserName" type="text"
+ placeholder="Input kafkaConfig parserName"
class="form-control"/>
<small class="help-block"
- ng-show="!form.cube_streaming_form.timeout.$error.required && form.cube_streaming_form.timeout.$invalid && (form.cube_streaming_form.timeout.$dirty||form.cube_streaming_form.$submitted)">
- Kafka timeout is invalid.
+ ng-show="!form.cube_streaming_form.parserName.$error.required && form.cube_streaming_form.parserName.$invalid && (form.cube_streaming_form.parserName.$dirty||form.cube_streaming_form.$submitted)">
+ Kafka parser name is invalid.
</small>
<small class="help-block"
- ng-show="form.cube_streaming_form.timeout.$error.required && (form.cube_streaming_form.timeout.$dirty||form.cube_streaming_form.$submitted)">
- Kafka timeout is required.
+ ng-show="form.cube_streaming_form.parserName.$error.required && (form.cube_streaming_form.parserName.$dirty||form.cube_streaming_form.$submitted)">
+ Kafka parser name is required.
</small>
- <span ng-if="state.mode=='view'">{{kafkaMeta.timeout}}</span>
+ <span ng-if="state.mode=='view'">{{kafkaMeta.parserName}}</span>
</div>
</div>
</div>
-
- <div class="form-group middle-popover" ng-class="{'required':state.mode=='edit'}">
+ <div class="form-group middle-popover" ng-if="state.mode=='edit'&&state.target!=='kfkConfig'" ng-class="{'required':state.mode=='edit'}">
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right">
- <b>Buffer Size</b>
- <i class="fa fa-info-circle" kylinpopover placement="right" title="Buffer Size" template="BufferSizecTip.html"></i>
+ <b>Parser Timestamp Field</b>
+ <i class="fa fa-info-circle" kylinpopover placement="right" title="Parser Timestamp Field" template="ParserTimestampFieldsTip.html"></i>
</label>
- <div class="col-xs-12 col-sm-6"
- ng-class="{'has-error':form.cube_streaming_form.bufferSize.$invalid && (form.cube_streaming_form.bufferSize.$dirty||form.cube_streaming_form.$submitted)}">
- <input ng-if="state.mode=='edit'" name="bufferSize" required ng-model="kafkaMeta.bufferSize" type="text"
- placeholder="Input kafkaConfig bufferSize"
- ng-pattern="/^\+?[1-9][0-9]*$/"
- class="form-control"/>
- <small class="help-block"
- ng-show="!form.cube_streaming_form.bufferSize.$error.required && form.cube_streaming_form.bufferSize.$invalid && (form.cube_streaming_form.bufferSize.$dirty||form.cube_streaming_form.$submitted)">
- Kafka bufferSize is invalid.
- </small>
- <small class="help-block"
- ng-show="form.cube_streaming_form.bufferSize.$error.required && (form.cube_streaming_form.bufferSize.$dirty||form.cube_streaming_form.$submitted)">
- Kafka bufferSize is required.
- </small>
- <span ng-if="state.mode=='view'">{{kafkaMeta.bufferSize}}</span>
+ <div class="col-xs-12 col-sm-6 streamingParserCtr">
+ <select chosen ng-model="streamingCfg.parseTsColumn"
+ ng-options="column as column for column in streamingCfg.columnOptions "
+ ng-change="streamingTsColUpdate()"
+ data-placeholder="select a column"
+ class="chosen-select">
+ <option value=""></option>
+ </select>
</div>
</div>
</div>
-
<div class="form-group middle-popover" ng-class="{'required':state.mode=='edit'}">
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right">
- <b>Margin</b>
- <i class="fa fa-info-circle" kylinpopover placement="right" title="Margin" template="MarginTip.html"></i>
+ <b>Parser Properties</b>
+ <i class="fa fa-info-circle" kylinpopover placement="right" title="Parser Properties" template="ParserPropertiesTip.html"></i>
</label>
<div class="col-xs-12 col-sm-6"
- ng-class="{'has-error':form.cube_streaming_form.margin.$invalid && (form.cube_streaming_form.margin.$dirty||form.cube_streaming_form.$submitted)}">
- <input ng-if="state.mode=='edit'" name="margin" required ng-model="kafkaMeta.margin" type="text"
- placeholder="Input kafkaConfig margin"
- ng-pattern="/^\+?[1-9][0-9]*$/"
+ ng-class="{'has-error':form.cube_streaming_form.parserProperties.$invalid && (form.cube_streaming_form.parserProperties.$dirty||form.cube_streaming_form.$submitted)}">
+ <input ng-if="state.mode=='edit'" name="parserProperties" required ng-model="kafkaMeta.parserProperties" type="text"
+ placeholder="configA=1;configB=2"
class="form-control"/>
<small class="help-block"
- ng-show="!form.cube_streaming_form.margin.$error.required && form.cube_streaming_form.margin.$invalid && (form.cube_streaming_form.margin.$dirty||form.cube_streaming_form.$submitted)">
- Kafka margin is invalid.
+ ng-show="!form.cube_streaming_form.parserProperties.$error.required && form.cube_streaming_form.parserProperties.$invalid && (form.cube_streaming_form.parserProperties.$dirty||form.cube_streaming_form.$submitted)">
+ Parser properties is invalid.
</small>
<small class="help-block"
- ng-show="form.cube_streaming_form.margin.$error.required && (form.cube_streaming_form.margin.$dirty||form.cube_streaming_form.$submitted)">
- Kafka margin is required.
+ ng-show="form.cube_streaming_form.parserProperties.$error.required && (form.cube_streaming_form.parserProperties.$dirty||form.cube_streaming_form.$submitted)">
+ Parser properties is required.
</small>
- <span ng-if="state.mode=='view'">{{kafkaMeta.margin}}</span>
+ <span ng-if="state.mode=='view'">{{kafkaMeta.parserProperties}}</span>
</div>
</div>
</div>
</accordion-group>
</accordion>
-
<hr/>
+ <!--Advanced setting-->
<accordion>
-
- <accordion-group is-open="state.isParserHeaderOpen">
+ <accordion-group is-open="state.isStreamingAdOpen" ng-init="state.isStreamingAdOpen">
<accordion-heading>
- Parser Setting
+ Advanced Setting
<i class="pull-right glyphicon"
- ng-class="{'glyphicon-chevron-down': state.isParserHeaderOpen, 'glyphicon-chevron-right': !state.isParserHeaderOpen}"></i>
+ ng-class="{'glyphicon-chevron-down': state.isStreamingAdOpen, 'glyphicon-chevron-right': !state.isStreamingAdOpen}"></i>
</accordion-heading>
<div class="form-group middle-popover" ng-class="{'required':state.mode=='edit'}">
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right">
- <b>Parser Name</b>
- <i class="fa fa-info-circle" kylinpopover placement="right" title="Parser Name" template="ParserName.html"></i>
+ <b>Timeout</b>
+ <i class="fa fa-info-circle" kylinpopover placement="right" title="Timeout" template="TimeoutTip.html"></i>
</label>
<div class="col-xs-12 col-sm-6"
- ng-class="{'has-error':form.cube_streaming_form.parserName.$invalid && (form.cube_streaming_form.parserName.$dirty||form.cube_streaming_form.$submitted)}">
- <input ng-if="state.mode=='edit'" name="parserName" required ng-model="kafkaMeta.parserName" type="text"
- placeholder="Input kafkaConfig parserName"
+ ng-class="{'has-error':form.cube_streaming_form.timeout.$invalid && (form.cube_streaming_form.timeout.$dirty||form.cube_streaming_form.$submitted)}">
+ <input ng-if="state.mode=='edit'" name="timeout" required ng-model="kafkaMeta.timeout" type="text"
+ placeholder="Input kafkaConfig timeout"
+ ng-pattern="/^\+?[1-9][0-9]*$/"
class="form-control"/>
<small class="help-block"
- ng-show="!form.cube_streaming_form.parserName.$error.required && form.cube_streaming_form.parserName.$invalid && (form.cube_streaming_form.parserName.$dirty||form.cube_streaming_form.$submitted)">
- Kafka parser name is invalid.
+ ng-show="!form.cube_streaming_form.timeout.$error.required && form.cube_streaming_form.timeout.$invalid && (form.cube_streaming_form.timeout.$dirty||form.cube_streaming_form.$submitted)">
+ Kafka timeout is invalid.
</small>
<small class="help-block"
- ng-show="form.cube_streaming_form.parserName.$error.required && (form.cube_streaming_form.parserName.$dirty||form.cube_streaming_form.$submitted)">
- Kafka parser name is required.
+ ng-show="form.cube_streaming_form.timeout.$error.required && (form.cube_streaming_form.timeout.$dirty||form.cube_streaming_form.$submitted)">
+ Kafka timeout is required.
</small>
- <span ng-if="state.mode=='view'">{{kafkaMeta.parserName}}</span>
+ <span ng-if="state.mode=='view'">{{kafkaMeta.timeout}}</span>
</div>
</div>
</div>
- <div class="form-group middle-popover" ng-if="state.mode=='edit'&&state.target!=='kfkConfig'" ng-class="{'required':state.mode=='edit'}">
- <div class="row">
- <label class="col-xs-12 col-sm-3 control-label no-padding-right">
- <b>Parser Timestamp Field</b>
- <i class="fa fa-info-circle" kylinpopover placement="right" title="Parser Timestamp Field" template="ParserTimestampFieldsTip.html"></i>
- </label>
- <div class="col-xs-12 col-sm-6 streamingParserCtr">
- <select chosen ng-model="streamingCfg.parseTsColumn"
- ng-options="column as column for column in streamingCfg.columnOptions "
- ng-change="streamingTsColUpdate()"
- data-placeholder="select a column"
- class="chosen-select">
- <option value=""></option>
- </select>
- </div>
- </div>
- </div>
<div class="form-group middle-popover" ng-class="{'required':state.mode=='edit'}">
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right">
- <b>Parser Properties</b>
- <i class="fa fa-info-circle" kylinpopover placement="right" title="Parser Properties" template="ParserPropertiesTip.html"></i>
+ <b>Buffer Size</b>
+ <i class="fa fa-info-circle" kylinpopover placement="right" title="Buffer Size" template="BufferSizecTip.html"></i>
</label>
<div class="col-xs-12 col-sm-6"
- ng-class="{'has-error':form.cube_streaming_form.parserProperties.$invalid && (form.cube_streaming_form.parserProperties.$dirty||form.cube_streaming_form.$submitted)}">
- <input ng-if="state.mode=='edit'" name="parserProperties" required ng-model="kafkaMeta.parserProperties" type="text"
- placeholder="configA=1;configB=2"
+ ng-class="{'has-error':form.cube_streaming_form.bufferSize.$invalid && (form.cube_streaming_form.bufferSize.$dirty||form.cube_streaming_form.$submitted)}">
+ <input ng-if="state.mode=='edit'" name="bufferSize" required ng-model="kafkaMeta.bufferSize" type="text"
+ placeholder="Input kafkaConfig bufferSize"
+ ng-pattern="/^\+?[1-9][0-9]*$/"
class="form-control"/>
<small class="help-block"
- ng-show="!form.cube_streaming_form.parserProperties.$error.required && form.cube_streaming_form.parserProperties.$invalid && (form.cube_streaming_form.parserProperties.$dirty||form.cube_streaming_form.$submitted)">
- Parser properties is invalid.
+ ng-show="!form.cube_streaming_form.bufferSize.$error.required && form.cube_streaming_form.bufferSize.$invalid && (form.cube_streaming_form.bufferSize.$dirty||form.cube_streaming_form.$submitted)">
+ Kafka bufferSize is invalid.
</small>
<small class="help-block"
- ng-show="form.cube_streaming_form.parserProperties.$error.required && (form.cube_streaming_form.parserProperties.$dirty||form.cube_streaming_form.$submitted)">
- Parser properties is required.
+ ng-show="form.cube_streaming_form.bufferSize.$error.required && (form.cube_streaming_form.bufferSize.$dirty||form.cube_streaming_form.$submitted)">
+ Kafka bufferSize is required.
</small>
- <span ng-if="state.mode=='view'">{{kafkaMeta.parserProperties}}</span>
+ <span ng-if="state.mode=='view'">{{kafkaMeta.bufferSize}}</span>
</div>
</div>
</div>
+
</accordion-group>
</accordion>
+
+
</div>
</form>
</div>
[6/6] kylin git commit: tmp
Posted by li...@apache.org.
tmp
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c1e6ff68
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c1e6ff68
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c1e6ff68
Branch: refs/heads/KYLIN-1971
Commit: c1e6ff6807097324a8c303796217a34c37ad08a2
Parents: 5cdd225
Author: Li Yang <li...@apache.org>
Authored: Wed Oct 26 18:24:15 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Oct 26 18:24:15 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeInstance.java | 6 ++++++
.../java/org/apache/kylin/cube/model/CubeDesc.java | 14 ++++++++++++--
.../org/apache/kylin/metadata/model/JoinDesc.java | 2 +-
.../org/apache/kylin/metadata/model/TblColRef.java | 4 ++--
.../kylin/metadata/realization/IRealization.java | 3 +++
.../kylin/storage/hybrid/HybridInstance.java | 17 +++++++++++++++++
.../org/apache/kylin/query/ITKylinQueryTest.java | 6 +++---
.../apache/kylin/query/routing/ModelChooser.java | 12 +++++++++++-
8 files changed, 55 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 2ccdde7..6bd3826 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.metadata.model.LookupDesc;
@@ -395,6 +396,11 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
@Override
+ public Set<ColumnDesc> getAllColumnDescs() {
+ return getDescriptor().listAllColumnDescs();
+ }
+
+ @Override
public long getDateRangeStart() {
List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 5c73f21..0c0b4f3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -54,6 +54,7 @@ import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IEngineAware;
@@ -159,8 +160,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
@JsonProperty("override_kylin_properties")
private LinkedHashMap<String, String> overrideKylinProps = new LinkedHashMap<String, String>();
- private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>();
- private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<TblColRef>();
+ private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>();
+ private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>();
+ private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>();
private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap();
private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap();
@@ -191,6 +193,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
public Set<TblColRef> listAllColumns() {
return allColumns;
}
+
+ public Set<ColumnDesc> listAllColumnDescs() {
+ return allColumnDescs;
+ }
/**
* @return dimension columns including derived, BUT NOT measures
@@ -549,6 +555,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey columns count (%d) doesn't match dimensions columns count (%d)", rowkey.getRowKeyColumns().length, dimCols.size());
initDictionaryDesc();
+
+ for (TblColRef col : allColumns) {
+ allColumnDescs.add(col.getColumnDesc());
+ }
}
public void validateAggregationGroups() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index 04fbf62..5beda0a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -154,7 +154,7 @@ public class JoinDesc {
int match = 0;
for (int i = 0; i < a.length; i++) {
for (int j = 0; j < b.length; j++) {
- if (a[i].equals(b[j])) {
+ if (a[i].getColumnDesc().equals(b[j].getColumnDesc())) {
match++;
break;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 8841ee5..f44972c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -183,8 +183,8 @@ public class TblColRef implements Serializable {
return false;
if (!StringUtils.equals(column.getName(), other.column.getName()))
return false;
-// if ((table == null ? other.table == null : table.equals(other.table)) == false)
-// return false;
+ if ((table == null ? other.table == null : table.equals(other.table)) == false)
+ return false;
return true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index 343ec99..f1770d7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -43,6 +44,8 @@ public interface IRealization extends IStorageAware {
public String getFactTable();
public Set<TblColRef> getAllColumns();
+
+ public Set<ColumnDesc> getAllColumnDescs();
public List<TblColRef> getAllDimensions();
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 57d14d5..d7007ac 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -68,6 +69,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
private volatile IRealization[] realizations = null;
private List<TblColRef> allDimensions = null;
private Set<TblColRef> allColumns = null;
+ private Set<ColumnDesc> allColumnDescs = null;
private List<MeasureDesc> allMeasures = null;
private long dateRangeStart;
private long dateRangeEnd;
@@ -138,6 +140,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
allDimensions = Lists.newArrayList(dimensions);
allColumns = columns;
+ allColumnDescs = asColumnDescs(allColumns);
allMeasures = Lists.newArrayList(measures);
Collections.sort(realizationList, new Comparator<IRealization>() {
@@ -166,6 +169,14 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
}
}
+ private Set<ColumnDesc> asColumnDescs(Set<TblColRef> columns) {
+ LinkedHashSet<ColumnDesc> result = new LinkedHashSet<>();
+ for (TblColRef col : columns) {
+ result.add(col.getColumnDesc());
+ }
+ return result;
+ }
+
@Override
public CapabilityResult isCapable(SQLDigest digest) {
CapabilityResult result = new CapabilityResult();
@@ -210,6 +221,12 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
}
@Override
+ public Set<ColumnDesc> getAllColumnDescs() {
+ init();
+ return allColumnDescs;
+ }
+
+ @Override
public List<MeasureDesc> getMeasures() {
init();
return allMeasures;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 2ec5324..54f02aa 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -45,7 +45,7 @@ import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
-@Ignore("KylinQueryTest is contained by ITCombinationTest")
+//@Ignore("KylinQueryTest is contained by ITCombinationTest")
public class ITKylinQueryTest extends KylinTestBase {
@Rule
@@ -150,11 +150,11 @@ public class ITKylinQueryTest extends KylinTestBase {
}
- @Ignore
+ //@Ignore
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql";
+ String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_subquery/query11.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e6ff68/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index 4667f4a..fcfcfbb 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -26,10 +26,12 @@ import java.util.TreeMap;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.LookupDesc;
import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.relnode.OLAPContext;
@@ -110,7 +112,7 @@ public class ModelChooser {
for (IRealization real : realizations) {
if (real.isReady() == false)
continue;
- if (real.getAllColumns().containsAll(context.allColumns) == false)
+ if (containsAll(real.getAllColumnDescs(), context.allColumns) == false)
continue;
if (RemoveBlackoutRealizationsRule.accept(real) == false)
continue;
@@ -143,6 +145,14 @@ public class ModelChooser {
return result;
}
+ private static boolean containsAll(Set<ColumnDesc> allColumnDescs, Set<TblColRef> allColumns) {
+ for (TblColRef col : allColumns) {
+ if (allColumnDescs.contains(col.getColumnDesc()) == false)
+ return false;
+ }
+ return true;
+ }
+
private static void fixModel(OLAPContext context, DataModelDesc model, Map<String, String> aliasMap) {
for (OLAPTableScan tableScan : context.allTableScans) {
tableScan.fixColumnRowTypeWithModel(model, aliasMap);
[3/6] kylin git commit: KYLIN-2126 use column name as default
dimension name when auto generate dimension for lookup table
Posted by li...@apache.org.
KYLIN-2126 use column name as default dimension name when auto generate dimension for lookup table
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6350e31f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6350e31f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6350e31f
Branch: refs/heads/KYLIN-1971
Commit: 6350e31f4267f456706f8964c4875af90c170f99
Parents: 92c823b
Author: Jason <ji...@163.com>
Authored: Tue Oct 25 21:00:01 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Tue Oct 25 21:00:01 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeDimensions.js | 6 +++---
webapp/app/partials/cubeDesigner/dimensions.html | 14 +-------------
2 files changed, 4 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6350e31f/webapp/app/js/controllers/cubeDimensions.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeDimensions.js b/webapp/app/js/controllers/cubeDimensions.js
index 5525fe4..ba71c96 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -96,7 +96,7 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
cols2[k].isLookup = true;
// Default not selected and not disabled.
- lookupSelectAvailable[cols2[k].name] = {name:cols2[k].table+"_derived",selected: false, disabled: false};
+ lookupSelectAvailable[cols2[k].name] = {name:cols2[k].name,selected: false, disabled: false};
}
$scope.availableColumns[lookups[j].table] = cols2;
@@ -302,7 +302,7 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
if(dim.table==$scope.metaModel.model.fact_table){
$scope.selectedColumns[dim.table][colName] = {name:colName,selected: false, disabled: false};
}else{
- $scope.selectedColumns[dim.table][colName] = {name:dim.table+"_derived",selected: false, disabled: false};
+ $scope.selectedColumns[dim.table][colName] = {name:colName,selected: false, disabled: false};
}
});
};
@@ -400,7 +400,7 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
if(table==$scope.metaModel.model.fact_table){
$scope.selectedColumns[table][name].name=name;
}else{
- $scope.selectedColumns[table][name].name=table+"_derived";
+ $scope.selectedColumns[table][name].name=name;
}
}else{
if($scope.metaModel.model.fact_table!=table){
http://git-wip-us.apache.org/repos/asf/kylin/blob/6350e31f/webapp/app/partials/cubeDesigner/dimensions.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/dimensions.html b/webapp/app/partials/cubeDesigner/dimensions.html
index 3a2efb6..0c4236e 100644
--- a/webapp/app/partials/cubeDesigner/dimensions.html
+++ b/webapp/app/partials/cubeDesigner/dimensions.html
@@ -276,7 +276,7 @@
</label>
</td>
<td >
- <input type="text" ng-model="selectedColumns[table][col.name].name" placeholder={{table}}_derived ng-disabled="!selectedColumns[table][col.name].selected" style="width:90%;">
+ <input type="text" ng-model="selectedColumns[table][col.name].name" ng-disabled="!selectedColumns[table][col.name].selected" style="width:90%;">
</td>
<td >
{{col.name}}
@@ -295,18 +295,6 @@
</div>
</div>
</div>
-
- <!--Tips-->
- <div class="col-xs-4">
- <div class="box box-solid">
- <div class="box-header">
- <h4 class="box-title"></h4>
- </div>
- <div class="box-body">
-
- </div>
- </div>
- </div>
</div>
</div>
<div class="modal-footer">