You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2020/02/21 03:21:29 UTC
[kylin] branch master updated (d0d817c -> 93c05df)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from d0d817c minor, externalizable classes should have no-arguments constructors
new 4a484f8 KYLIN-4336 Global domain dict for MR build engine
new 58ac324 KYLIN-4336 Global domain dict for MR build engine ,fix unit test error
new eaa1cdb KYLIN-4337 Global domain dict for Spark build engine
new 47e2ebe KYLIN-4338 Web page modification to support global domain dict
new 0a25dd2 Fix bug for spark build.
new 93c05df KYLIN-4336 Code review
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/kylin/cube/CubeManager.java | 15 +-
.../java/org/apache/kylin/cube/model/CubeDesc.java | 34 ++++-
.../CubeDescTiretreeGlobalDomainDictUtil.java | 170 +++++++++++++++++++++
.../apache/kylin/cube/model/DictionaryDesc.java | 42 +++++
.../cube/model/validation/rule/DictionaryRule.java | 12 +-
.../kylin/measure/bitmap/BitmapMeasureType.java | 8 +-
.../apache/kylin/metadata/model/DataModelDesc.java | 20 ++-
.../kylin/metadata/model/DataModelManagerTest.java | 7 -
.../kylin/engine/mr/common/AbstractHadoopJob.java | 4 +
.../apache/kylin/engine/spark/SparkExecutable.java | 6 +
webapp/app/js/controllers/cubeAdvanceSetting.js | 41 +++--
.../partials/cubeDesigner/advanced_settings.html | 38 ++++-
12 files changed, 357 insertions(+), 40 deletions(-)
create mode 100644 core-cube/src/main/java/org/apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java
[kylin] 02/06: KYLIN-4336 Global domain dict for MR build engine ,
fix unit test error
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 58ac324840742803103355b8a26297a35e264a77
Author: wangxiaojing <wa...@didichuxing.com>
AuthorDate: Mon Jan 13 20:32:54 2020 +0800
KYLIN-4336 Global domain dict for MR build engine ,fix unit test error
---
.../java/org/apache/kylin/metadata/model/DataModelManagerTest.java | 7 -------
1 file changed, 7 deletions(-)
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelManagerTest.java
index ee75302..ff6b3bc 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelManagerTest.java
@@ -78,13 +78,6 @@ public class DataModelManagerTest extends LocalFileMetadataTestCase {
private void assertSnowflakeQuality(DataModelDesc model) {
Assert.assertNotNull(model);
- try {
- model.findTable("TEST_COUNTRY");
- Assert.fail();
- } catch (IllegalArgumentException ex) {
- // excepted
- }
-
Assert.assertNotNull(model.findTable("BUYER_COUNTRY"));
Assert.assertNotNull(model.findTable("SELLER_COUNTRY"));
Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME"));
[kylin] 05/06: Fix bug for spark build.
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 0a25dd2468f463124137949ba86a7fef9edbee22
Author: XiaoxiangYu <hi...@126.com>
AuthorDate: Thu Feb 20 00:55:38 2020 +0800
Fix bug for spark build.
---
.../apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java
index 999fcae..326ce53 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java
@@ -51,7 +51,7 @@ public class CubeDescTiretreeGlobalDomainDictUtil {
String globalResumeDictPath = null;
List<GlobalDict> globalDicts = cubeDesc.listDomainDict();
DataModelManager metadataManager = DataModelManager.getInstance(config);
- CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ CubeManager cubeManager = CubeManager.getInstance(config);
for (GlobalDict dict : globalDicts) {
if (dict.getSrc().getIdentity().equalsIgnoreCase(tblColRef.getIdentity())) {
String model = dict.getModel();
[kylin] 04/06: KYLIN-4338 Web page modification to support global
domain dict
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 47e2ebebff4d987a767c84c969d2d3e035ad3b93
Author: wangxiaojing <wa...@didichuxing.com>
AuthorDate: Mon Jan 13 21:11:07 2020 +0800
KYLIN-4338 Web page modification to support global domain dict
---
webapp/app/js/controllers/cubeAdvanceSetting.js | 35 ++++++++++++++------
.../partials/cubeDesigner/advanced_settings.html | 38 ++++++++++++++++++----
2 files changed, 57 insertions(+), 16 deletions(-)
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 1b7bcd1..4b6252e 100755
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -177,12 +177,14 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
}
- $scope.isReuse=false;
+ $scope.isReuse=1;
$scope.addNew=false;
$scope.newDictionaries = {
"column":null,
"builder": null,
- "reuse": null
+ "reuse": null,
+ "model": null,
+ "cube": null
}
$scope.initUpdateDictionariesStatus = function(){
@@ -198,11 +200,13 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
$scope.updateDictionariesStatus.isEdit = true;
$scope.addNew=true;
$scope.updateDictionariesStatus.editIndex = index;
- if(dictionaries.builder==null){
- $scope.isReuse=true;
+ if(dictionaries.builder==null && dictionaries.model==null){
+ $scope.isReuse=2;
+ }else if (dictionaries.model!=null){
+ $scope.isReuse=3;
}
else{
- $scope.isReuse=false;
+ $scope.isReuse=1;
}
}
else{
@@ -231,7 +235,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
$scope.initUpdateDictionariesStatus();
$scope.nextDictionariesInit();
$scope.addNew = !$scope.addNew;
- $scope.isReuse = false;
+ $scope.isReuse = 1;
return true;
};
@@ -240,7 +244,9 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
$scope.nextDic = {
"coiumn":null,
"builder":null,
- "reuse":null
+ "reuse":null,
+ "model":null,
+ "cube":null
}
}
@@ -261,16 +267,25 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
$scope.clearNewDictionaries = function (){
$scope.newDictionaries = null;
- $scope.isReuse=false;
+ $scope.isReuse=1;
$scope.initUpdateDictionariesStatus();
$scope.nextDictionariesInit();
$scope.addNew=!$scope.addNew;
}
- $scope.change = function (){
+ $scope.change = function (type){
$scope.newDictionaries.builder=null;
$scope.newDictionaries.reuse=null;
- $scope.isReuse=!$scope.isReuse;
+ $scope.newDictionaries.domain=null;
+ $scope.newDictionaries.model=null;
+ $scope.newDictionaries.cube=null;
+ if(type == 'domain'){
+ $scope.isReuse=3;
+ }else if (type == 'builder'){
+ $scope.isReuse=1;
+ }else if (type == 'reuse'){
+ $scope.isReuse=2;
+ }
}
$scope.removeElement = function(arr,element){
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index fbd8cdc..94c7d44 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -412,7 +412,7 @@
</td>
<td>
<!--Reuse-->
- <span>{{dictionaries.reuse}}</span>
+ <span>{{dictionaries.model}}.{{dictionaries.cube}}.{{dictionaries.reuse}}</span>
</td>
<td ng-if="state.mode=='edit'">
<!--Edit Button -->
@@ -444,7 +444,7 @@
</div>
<div class="box-body">
<div class="row">
- <div class="col-xs-8">
+ <div class="col-xs-12">
<!--Column-->
<div class="form-group middle-popover">
<div class="row">
@@ -462,15 +462,20 @@
<div class="form-group">
<div class="row" style="margin-left:5px;">
<label class="radio-inline ng-binding ng-scope">
- <input type="radio" name="reuse" ng-value=false ng-model="isReuse" ng-change="change()"/><b>Builder Class</b>
+ <input type="radio" name="reuse" ng-value=1 ng-model="isReuse" ng-change="change('builder')"/><b>Builder Class</b>
</label>
<label class="radio-inline ng-binding ng-scope">
- <input type="radio" name="reuse" ng-value=true ng-model="isReuse" ng-change="change()"/><b>Reuse</b>
+ <input type="radio" name="reuse" ng-value=2 ng-model="isReuse" ng-change="change('reuse')"/><b>Reuse Self</b>
</label>
+ <!-- tiretree add-->
+ <label class="radio-inline ng-binding ng-scope">
+ <input type="radio" name="reuse" ng-value=3 ng-model="isReuse" ng-change="change('domain')"/><b>Reuse Global Domain</b>
+ </label>
+ <!-- tiretree end-->
</div>
</div>
<!--Builder-->
- <div class="form-group" ng-if="!isReuse" >
+ <div class="form-group" ng-if="isReuse==1" >
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b>Builder Class</b></label>
<div class="col-xs-12 col-sm-6">
@@ -484,7 +489,7 @@
</div>
</div>
<!--Reuse-->
- <div class="form-group middle-popover" ng-if="isReuse" >
+ <div class="form-group middle-popover" ng-if="isReuse==2" >
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b>Reuse</b></label>
<div class="col-xs-12 col-sm-6">
@@ -497,6 +502,27 @@
</div>
</div>
</div>
+ <!-- Tiretree global dic model cube table.colname start-->
+ <div class="form-group middle-popover" ng-if="isReuse==3" >
+ <div class="row">
+ <label class="col-xs-6 col-sm-4 control-label no-padding-right" ng-class="{'has-error':newDictionaries.model==''}">
+ <b ng-if="state.mode=='view'">{{dictionaries.model}}</b>
+ <input ng-model="newDictionaries.model" placeholder="Reuse Model Name" required class="form-control" ng-if="state.mode=='edit'"/>
+ <small style="font-weight: normal !important;" class="help-block" ng-show="newDictionaries.model==''">Reuse Model Name is required.</small>
+ </label>
+ <div class="col-xs-6 col-sm-4" ng-class="{'has-error':newDictionaries.cube==''}">
+ <input ng-model="newDictionaries.cube" placeholder="Reuse Cube Name" class="form-control" ng-if="state.mode=='edit'" required/>
+ <small class="help-block" ng-show="newDictionaries.cube==''">Reuse Cube Name is required.</small>
+ <span ng-if="state.mode=='view'">{{dictionaries.cube}}</span>
+ </div>
+ <div class="col-xs-12 col-sm-4" ng-class="{'has-error':newDictionaries.reuse==''}">
+ <input ng-model="newDictionaries.reuse" placeholder="Reuse table.column" class="form-control" ng-if="state.mode=='edit'" required/>
+ <small class="help-block" ng-show="newDictionaries.reuse==''">Reuse table.column is required.</small>
+ <span ng-if="state.mode=='view'">{{dictionaries.reuse}}</span>
+ </div>
+ </div>
+ </div>
+ <!--end -->
</div>
</div>
</div>
[kylin] 01/06: KYLIN-4336 Global domain dict for MR build engine
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4a484f861550f07417e5890bc8478f182f922052
Author: wangxiaojing <wa...@didichuxing.com>
AuthorDate: Mon Jan 13 19:39:31 2020 +0800
KYLIN-4336 Global domain dict for MR build engine
---
.../java/org/apache/kylin/cube/CubeManager.java | 15 +-
.../java/org/apache/kylin/cube/model/CubeDesc.java | 26 ++++
.../CubeDescTiretreeGlobalDomainDictUtil.java | 164 +++++++++++++++++++++
.../apache/kylin/cube/model/DictionaryDesc.java | 42 ++++++
.../cube/model/validation/rule/DictionaryRule.java | 12 +-
.../kylin/measure/bitmap/BitmapMeasureType.java | 8 +-
.../apache/kylin/metadata/model/DataModelDesc.java | 20 ++-
.../kylin/engine/mr/common/AbstractHadoopJob.java | 4 +
8 files changed, 278 insertions(+), 13 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index d057982..7a44f60 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -27,6 +27,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -45,6 +46,7 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeDescTiretreeGlobalDomainDictUtil;
import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.cube.model.SnapshotTableDesc;
import org.apache.kylin.dict.DictionaryInfo;
@@ -1185,9 +1187,20 @@ public class CubeManager implements IRealizationProvider {
@SuppressWarnings("unchecked")
public Dictionary<String> getDictionary(CubeSegment cubeSeg, TblColRef col) {
DictionaryInfo info = null;
+ String dictResPath = null;
try {
DictionaryManager dictMgr = getDictionaryManager();
- String dictResPath = cubeSeg.getDictResPath(col);
+
+ //tiretree global domain dic
+ List<CubeDescTiretreeGlobalDomainDictUtil.GlobalDict> globalDicts = cubeSeg.getCubeDesc().listDomainDict();
+ if (!globalDicts.isEmpty()) {
+ dictResPath = CubeDescTiretreeGlobalDomainDictUtil.globalReuseDictPath(cubeSeg.getConfig(), col, cubeSeg.getCubeDesc());
+ }
+
+ if (Objects.isNull(dictResPath)){
+ dictResPath = cubeSeg.getDictResPath(col);
+ }
+
if (dictResPath == null)
return null;
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f5bb427..7c16d33 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -40,6 +40,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
@@ -1375,6 +1376,13 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
result.remove(dictDesc.getColumnRef());
result.add(dictDesc.getResuseColumnRef());
}
+
+ //tiretree global domain dic
+ if (Objects.isNull(dictDesc.getResuseColumnRef()) && Objects.nonNull(dictDesc.getReuseColumn())) {
+ logger.info("tiretree global domain dic : column {} use tiretree global domain dic, reuse column {} ", dictDesc.getColumnRef(), dictDesc.getReuseColumn());
+ result.remove(dictDesc.getColumnRef());
+ }
+
}
}
@@ -1382,6 +1390,24 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
}
/**
+ * get tiretree global domain dic
+ *
+ * @return
+ */
+ public List<CubeDescTiretreeGlobalDomainDictUtil.GlobalDict> listDomainDict() {
+ List<CubeDescTiretreeGlobalDomainDictUtil.GlobalDict> dicts = new ArrayList<>();
+ if(dictionaries!=null && dictionaries.size()>0) {
+ for (DictionaryDesc dictionaryDesc : dictionaries) {
+ if (dictionaryDesc.isDomain()) {
+ dicts.add(new CubeDescTiretreeGlobalDomainDictUtil.GlobalDict(dictionaryDesc.getColumnRef(), dictionaryDesc.getReuseColumn(), dictionaryDesc.getCube(), dictionaryDesc.getModel()));
+ }
+ }
+ }
+ return dicts;
+ }
+
+
+ /**
* A column may reuse dictionary of another column, find the dict column, return same col if there's no reuse column
*/
public TblColRef getDictionaryReuseColumn(TblColRef col) {
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java
new file mode 100644
index 0000000..999fcae
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.model;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.source.SourceManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+public class CubeDescTiretreeGlobalDomainDictUtil {
+ private static final Logger logger = LoggerFactory.getLogger(CubeDescTiretreeGlobalDomainDictUtil.class);
+
+ /**
+ * get reuse global tiretree global dic path
+ * @param tblColRef
+ * @param cubeDesc
+ * @return
+ */
+ public static String globalReuseDictPath(KylinConfig config, TblColRef tblColRef, CubeDesc cubeDesc) {
+ String globalResumeDictPath = null;
+ List<GlobalDict> globalDicts = cubeDesc.listDomainDict();
+ DataModelManager metadataManager = DataModelManager.getInstance(config);
+ CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ for (GlobalDict dict : globalDicts) {
+ if (dict.getSrc().getIdentity().equalsIgnoreCase(tblColRef.getIdentity())) {
+ String model = dict.getModel();
+ String cube = dict.getCube();
+ logger.info("cube:{} column:{} tiretree global domain dic reuse model:{} cube{} column:{} ", cubeDesc.getName() , tblColRef.getName(), model, cube, dict.getDesc());
+
+ DataModelDesc dataModel = metadataManager.getDataModelDesc(model);
+ if (Objects.isNull(dataModel)) {
+ logger.error("get cube:{} column:{} tiretree global domain dic reuse DataModelDesc error", cubeDesc.getName(), tblColRef.getName());
+ return null;
+ }
+
+ CubeInstance cubeInstance = cubeManager.getCube(cube);
+ CubeSegment cubeSegment = cubeInstance.getLatestReadySegment();
+
+ TblColRef colRef = dataModel.findColumn(dict.getDesc());
+ if(Objects.isNull(colRef)){
+ logger.error("get cube:{} column:{} tiretree global domain dic TblColRef error");
+ return null;
+ }
+
+ globalResumeDictPath = cubeSegment.getDictResPath(colRef);
+
+ if (StringUtils.isBlank(globalResumeDictPath)) {
+ logger.error("get cube:{} column:{} tiretree global domain dic resume dict path error");
+ }
+ logger.error("get cube:{} column:{} tiretree global domain dic resume dict path is {}", globalResumeDictPath);
+ break;
+ }
+ }
+ return globalResumeDictPath;
+ }
+
+
+
+
+ /**
+ * add resuce global tiretree global dic for baseid job
+ * @param cubeDesc
+ * @param dumpList
+ */
+ public static void cuboidJob(CubeDesc cubeDesc, Set<String> dumpList) {
+ logger.info("cube {} start to add global domain dic", cubeDesc.getName());
+ CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelManager metadataManager =DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());
+
+ cubeManager.getCube(cubeDesc.getName());
+ List<GlobalDict> globalDicts = cubeDesc.listDomainDict();
+
+ for (GlobalDict dict : globalDicts) {
+ String cube = dict.getCube();
+ String model = dict.getModel();
+ logger.debug("cube {} column {} start to add global domain dic ,reuse {}.{}.{}", cubeDesc.getName(), dict.getSrc(), model, cube, dict.getDesc());
+ CubeInstance instance = cubeManager.getCube(cube);
+ logger.debug("cube {} column {} start to add global domain dic ,reuse cube{} dict", cubeDesc.getName(), dict.getSrc(), instance.getName());
+
+ // cube, model_desc, cube_desc, table
+ dumpList.add(instance.getResourcePath());
+ dumpList.add(instance.getDescriptor().getModel().getResourcePath());
+ dumpList.add(instance.getDescriptor().getResourcePath());
+ dumpList.add(instance.getProjectInstance().getResourcePath());
+
+ for (TableRef tableRef : instance.getDescriptor().getModel().getAllTables()) {
+ TableDesc table = tableRef.getTableDesc();
+ dumpList.add(table.getResourcePath());
+ dumpList.addAll(SourceManager.getMRDependentResources(table));
+ }
+
+ DataModelDesc dataModelDesc = metadataManager.getDataModelDesc(model);
+ logger.debug("cube {} column {} start to add global domain dic ,reuse model{} dict", cubeDesc.getName(), dict.getSrc(), dataModelDesc.getName());
+ TblColRef tblColRef = dataModelDesc.findColumn(dict.getDesc());
+ CubeSegment segment = instance.getLatestReadySegment();
+ logger.debug("cube {} column {} start to add global domain dic ,reuse mode:{} cube:{} segment:{} dict,tblColRef:{}", cubeDesc.getName(), dict.getSrc(), dataModelDesc.getName(), cube, segment.getName(), tblColRef.getIdentity());
+ if(segment.getDictResPath(tblColRef)!=null) {
+ dumpList.addAll(ImmutableList.of(segment.getDictResPath(tblColRef)));
+ }
+ }
+ }
+
+
+ public static class GlobalDict implements Serializable {
+ private TblColRef src;
+ private String desc;
+ private String cube;
+ private String model;
+
+ public GlobalDict(TblColRef src, String desc, String cube, String model) {
+ this.src = src;
+ this.desc = desc;
+ this.cube = cube;
+ this.model = model;
+ }
+
+ public TblColRef getSrc() {
+ return src;
+ }
+
+ public String getDesc() {
+ return desc;
+ }
+
+ public String getCube() {
+ return cube;
+ }
+
+ public String getModel() {
+ return model;
+ }
+ }
+}
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
index a700e10..2d1ba99 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
@@ -19,6 +19,7 @@
package org.apache.kylin.cube.model;
import java.util.Locale;
+import java.util.Objects;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -40,6 +41,15 @@ public class DictionaryDesc implements java.io.Serializable {
@JsonInclude(JsonInclude.Include.NON_NULL)
private String builderClass;
+ //for tiretree global domain dic
+ @JsonProperty("cube")
+ private String cube;
+
+ //for tiretree global domain dic
+ @JsonProperty("model")
+ private String model;
+
+
// computed content
private TblColRef colRef;
private TblColRef reuseColRef;
@@ -68,6 +78,38 @@ public class DictionaryDesc implements java.io.Serializable {
return builderClass;
}
+ public String getModel() {
+ return model;
+ }
+
+ public void setModel(String model) {
+ this.model = model;
+ }
+
+ public String getCube() {
+ return cube;
+ }
+
+ public void setCube(String cube) {
+ this.cube = cube;
+ }
+
+ public String getReuseColumn() {
+ return reuseColumn;
+ }
+
+ /**
+ * check if the col is tiretree global domain dic
+ * @return
+ */
+ public boolean isDomain() {
+ if (Objects.isNull(reuseColRef) && Objects.nonNull(reuseColumn)) {
+ return true;
+ }
+ return false;
+ }
+
+
// for test
public static DictionaryDesc create(String column, String reuseColumn, String builderClass) {
DictionaryDesc desc = new DictionaryDesc();
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
index df1316d..9023f28 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
@@ -32,6 +32,8 @@ import org.apache.kylin.cube.model.validation.ResultLevel;
import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.dict.GlobalDictionaryBuilder;
import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Validate Dictionary Settings:
@@ -43,6 +45,8 @@ import org.apache.kylin.metadata.model.TblColRef;
* </ul>
*/
public class DictionaryRule implements IValidatorRule<CubeDesc> {
+ private static final Logger logger = LoggerFactory.getLogger(DictionaryRule.class);
+
static final String ERROR_DUPLICATE_DICTIONARY_COLUMN = "Duplicated dictionary specification for column: ";
static final String ERROR_REUSE_BUILDER_BOTH_SET = "REUSE and BUILDER both set on dictionary for column: ";
static final String ERROR_REUSE_BUILDER_BOTH_EMPTY = "REUSE and BUILDER both empty on dictionary for column: ";
@@ -80,8 +84,12 @@ public class DictionaryRule implements IValidatorRule<CubeDesc> {
}
if (reuseCol == null && StringUtils.isEmpty(builderClass)) {
- context.addResult(ResultLevel.ERROR, ERROR_REUSE_BUILDER_BOTH_EMPTY + dictCol);
- return;
+ if(dictDesc.isDomain()) {
+ logger.info("() is tiretree global domain dic", dictCol);
+ }else{
+ context.addResult(ResultLevel.ERROR, ERROR_REUSE_BUILDER_BOTH_EMPTY + dictCol);
+ return;
+ }
}
if (StringUtils.isNotEmpty(builderClass) && builderClass.equalsIgnoreCase(GlobalDictionaryBuilder.class.getName()) && dimensionColumns.contains(dictCol) && rowKeyDesc.isUseDictionary(dictCol)) {
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
index 7134c6b..450a62a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
@@ -110,10 +110,10 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
}
int id;
- if (needDictionaryColumn(measureDesc.getFunction())) {
- TblColRef literalCol = measureDesc.getFunction().getParameter().getColRefs().get(0);
- Dictionary<String> dictionary = dictionaryMap.get(literalCol);
- id = dictionary.getIdFromValue(values[0]);
+ TblColRef literalCol = measureDesc.getFunction().getParameter().getColRefs().get(0);
+ if (needDictionaryColumn(measureDesc.getFunction()) && dictionaryMap.containsKey(literalCol)) {
+ Dictionary<String> dictionary = dictionaryMap.get(literalCol);
+ id = dictionary.getIdFromValue(values[0]);
} else {
id = Integer.parseInt(values[0]);
}
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 818afdf..12dd63f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -27,6 +27,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Queue;
import java.util.Set;
@@ -285,10 +286,16 @@ public class DataModelDesc extends RootPersistentEntity {
}
public TblColRef findColumn(String table, String column) throws IllegalArgumentException {
+ TblColRef result = null;
TableRef tableRef = findTable(table);
- TblColRef result = tableRef.getColumn(column.toUpperCase(Locale.ROOT));
- if (result == null)
- throw new IllegalArgumentException("Column not found by " + table + "." + column);
+ if (Objects.nonNull(tableRef)) {
+ result = tableRef.getColumn(column.toUpperCase(Locale.ROOT));;
+ }
+
+ if (result == null) {//tiretree global domain dic
+ logger.warn("table {} column {} not found in its's model {} , maybe it's a tiretree global domain dict. ", table, column, getName() );
+ }
+
return result;
}
@@ -310,8 +317,9 @@ public class DataModelDesc extends RootPersistentEntity {
}
}
- if (result == null)
- throw new IllegalArgumentException("Column not found by " + input);
+ if (result == null) {
+ logger.warn("Column {} not found in its's model {} , maybe it's a tiretree global domain dict. ", column, getName() );
+ }
return result;
}
@@ -320,7 +328,7 @@ public class DataModelDesc extends RootPersistentEntity {
public TableRef findTable(String table) throws IllegalArgumentException {
TableRef result = tableNameMap.get(table.toUpperCase(Locale.ROOT));
if (result == null) {
- throw new IllegalArgumentException("Table not found by " + table);
+ logger.warn("table {} not found in its's model {} , maybe it's a tiretree global domain dict. ", table, getName() );
}
return result;
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 5e49c76..fd4d413 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -69,6 +69,7 @@ import org.apache.kylin.common.util.StringSplitter;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDescTiretreeGlobalDomainDictUtil;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.model.TableDesc;
@@ -586,6 +587,9 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
if (ifStatsIncluded) {
dumpList.add(segment.getStatisticsResourcePath());
}
+ //tiretree global domain dic
+ CubeDescTiretreeGlobalDomainDictUtil.cuboidJob(segment.getCubeDesc(), dumpList);
+
dumpKylinPropsAndMetadata(segment.getProject(), dumpList, segment.getConfig(), conf);
}
[kylin] 06/06: KYLIN-4336 Code review
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 93c05dfc09f8fcce505c1ee34f77cd96f69f2988
Author: nichunen <ni...@apache.org>
AuthorDate: Thu Feb 20 21:01:36 2020 +0800
KYLIN-4336 Code review
---
.../java/org/apache/kylin/cube/model/CubeDesc.java | 16 ++++----
.../CubeDescTiretreeGlobalDomainDictUtil.java | 46 ++++++++++++----------
webapp/app/js/controllers/cubeAdvanceSetting.js | 24 ++++++-----
3 files changed, 49 insertions(+), 37 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 7c16d33..f0ec311 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -497,10 +497,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
if (!name.equals(cubeDesc.name))
return false;
- if (!modelName.equals(cubeDesc.modelName))
- return false;
+ return modelName.equals(cubeDesc.modelName);
- return true;
}
@Override
@@ -1034,7 +1032,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
whatsLeft.add(derCol);
}
if (whatsLeft.size() > 0) {
- infoList.add(new DeriveInfo(type, join, (TblColRef[]) whatsLeft.toArray(new TblColRef[whatsLeft.size()]),
+ infoList.add(new DeriveInfo(type, join, whatsLeft.toArray(new TblColRef[whatsLeft.size()]),
false));
}
}
@@ -1379,7 +1377,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
//tiretree global domain dic
if (Objects.isNull(dictDesc.getResuseColumnRef()) && Objects.nonNull(dictDesc.getReuseColumn())) {
- logger.info("tiretree global domain dic : column {} use tiretree global domain dic, reuse column {} ", dictDesc.getColumnRef(), dictDesc.getReuseColumn());
+ logger.info(
+ "tiretree global domain dic : column {} use tiretree global domain dic, reuse column {} ",
+ dictDesc.getColumnRef(), dictDesc.getReuseColumn());
result.remove(dictDesc.getColumnRef());
}
@@ -1396,17 +1396,17 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
*/
public List<CubeDescTiretreeGlobalDomainDictUtil.GlobalDict> listDomainDict() {
List<CubeDescTiretreeGlobalDomainDictUtil.GlobalDict> dicts = new ArrayList<>();
- if(dictionaries!=null && dictionaries.size()>0) {
+ if (dictionaries != null && dictionaries.size() > 0) {
for (DictionaryDesc dictionaryDesc : dictionaries) {
if (dictionaryDesc.isDomain()) {
- dicts.add(new CubeDescTiretreeGlobalDomainDictUtil.GlobalDict(dictionaryDesc.getColumnRef(), dictionaryDesc.getReuseColumn(), dictionaryDesc.getCube(), dictionaryDesc.getModel()));
+ dicts.add(new CubeDescTiretreeGlobalDomainDictUtil.GlobalDict(dictionaryDesc.getColumnRef(),
+ dictionaryDesc.getReuseColumn(), dictionaryDesc.getCube(), dictionaryDesc.getModel()));
}
}
}
return dicts;
}
-
/**
* A column may reuse dictionary of another column, find the dict column, return same col if there's no reuse column
*/
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java
index 326ce53..0746b9a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDescTiretreeGlobalDomainDictUtil.java
@@ -18,7 +18,11 @@
package org.apache.kylin.cube.model;
-import com.google.common.collect.ImmutableList;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
@@ -33,10 +37,7 @@ import org.apache.kylin.source.SourceManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Serializable;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
+import com.google.common.collect.ImmutableList;
public class CubeDescTiretreeGlobalDomainDictUtil {
private static final Logger logger = LoggerFactory.getLogger(CubeDescTiretreeGlobalDomainDictUtil.class);
@@ -56,11 +57,13 @@ public class CubeDescTiretreeGlobalDomainDictUtil {
if (dict.getSrc().getIdentity().equalsIgnoreCase(tblColRef.getIdentity())) {
String model = dict.getModel();
String cube = dict.getCube();
- logger.info("cube:{} column:{} tiretree global domain dic reuse model:{} cube{} column:{} ", cubeDesc.getName() , tblColRef.getName(), model, cube, dict.getDesc());
+ logger.info("cube:{} column:{} tiretree global domain dic reuse model:{} cube{} column:{} ",
+ cubeDesc.getName(), tblColRef.getName(), model, cube, dict.getDesc());
DataModelDesc dataModel = metadataManager.getDataModelDesc(model);
if (Objects.isNull(dataModel)) {
- logger.error("get cube:{} column:{} tiretree global domain dic reuse DataModelDesc error", cubeDesc.getName(), tblColRef.getName());
+ logger.error("get cube:{} column:{} tiretree global domain dic reuse DataModelDesc error",
+ cubeDesc.getName(), tblColRef.getName());
return null;
}
@@ -68,7 +71,7 @@ public class CubeDescTiretreeGlobalDomainDictUtil {
CubeSegment cubeSegment = cubeInstance.getLatestReadySegment();
TblColRef colRef = dataModel.findColumn(dict.getDesc());
- if(Objects.isNull(colRef)){
+ if (Objects.isNull(colRef)) {
logger.error("get cube:{} column:{} tiretree global domain dic TblColRef error");
return null;
}
@@ -78,16 +81,14 @@ public class CubeDescTiretreeGlobalDomainDictUtil {
if (StringUtils.isBlank(globalResumeDictPath)) {
logger.error("get cube:{} column:{} tiretree global domain dic resume dict path error");
}
- logger.error("get cube:{} column:{} tiretree global domain dic resume dict path is {}", globalResumeDictPath);
+ logger.error("get cube:{} column:{} tiretree global domain dic resume dict path is {}",
+ globalResumeDictPath);
break;
}
}
return globalResumeDictPath;
}
-
-
-
/**
* add resuce global tiretree global dic for baseid job
* @param cubeDesc
@@ -96,7 +97,7 @@ public class CubeDescTiretreeGlobalDomainDictUtil {
public static void cuboidJob(CubeDesc cubeDesc, Set<String> dumpList) {
logger.info("cube {} start to add global domain dic", cubeDesc.getName());
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- DataModelManager metadataManager =DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelManager metadataManager = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());
cubeManager.getCube(cubeDesc.getName());
List<GlobalDict> globalDicts = cubeDesc.listDomainDict();
@@ -104,9 +105,11 @@ public class CubeDescTiretreeGlobalDomainDictUtil {
for (GlobalDict dict : globalDicts) {
String cube = dict.getCube();
String model = dict.getModel();
- logger.debug("cube {} column {} start to add global domain dic ,reuse {}.{}.{}", cubeDesc.getName(), dict.getSrc(), model, cube, dict.getDesc());
+ logger.debug("cube {} column {} start to add global domain dic ,reuse {}.{}.{}", cubeDesc.getName(),
+ dict.getSrc(), model, cube, dict.getDesc());
CubeInstance instance = cubeManager.getCube(cube);
- logger.debug("cube {} column {} start to add global domain dic ,reuse cube{} dict", cubeDesc.getName(), dict.getSrc(), instance.getName());
+ logger.debug("cube {} column {} start to add global domain dic ,reuse cube{} dict", cubeDesc.getName(),
+ dict.getSrc(), instance.getName());
// cube, model_desc, cube_desc, table
dumpList.add(instance.getResourcePath());
@@ -121,17 +124,20 @@ public class CubeDescTiretreeGlobalDomainDictUtil {
}
DataModelDesc dataModelDesc = metadataManager.getDataModelDesc(model);
- logger.debug("cube {} column {} start to add global domain dic ,reuse model{} dict", cubeDesc.getName(), dict.getSrc(), dataModelDesc.getName());
+ logger.debug("cube {} column {} start to add global domain dic ,reuse model{} dict", cubeDesc.getName(),
+ dict.getSrc(), dataModelDesc.getName());
TblColRef tblColRef = dataModelDesc.findColumn(dict.getDesc());
- CubeSegment segment = instance.getLatestReadySegment();
- logger.debug("cube {} column {} start to add global domain dic ,reuse mode:{} cube:{} segment:{} dict,tblColRef:{}", cubeDesc.getName(), dict.getSrc(), dataModelDesc.getName(), cube, segment.getName(), tblColRef.getIdentity());
- if(segment.getDictResPath(tblColRef)!=null) {
+ CubeSegment segment = instance.getLatestReadySegment();
+ logger.debug(
+ "cube {} column {} start to add global domain dic ,reuse mode:{} cube:{} segment:{} dict,tblColRef:{}",
+ cubeDesc.getName(), dict.getSrc(), dataModelDesc.getName(), cube, segment.getName(),
+ tblColRef.getIdentity());
+ if (segment.getDictResPath(tblColRef) != null) {
dumpList.addAll(ImmutableList.of(segment.getDictResPath(tblColRef)));
}
}
}
-
public static class GlobalDict implements Serializable {
private TblColRef src;
private String desc;
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 4b6252e..1048839 100755
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -177,6 +177,12 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
}
+ var ReuseEnum = {
+ BUILD: 1,
+ SELF: 2,
+ DOMAIN: 3
+ };
+
$scope.isReuse=1;
$scope.addNew=false;
$scope.newDictionaries = {
@@ -201,12 +207,12 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
$scope.addNew=true;
$scope.updateDictionariesStatus.editIndex = index;
if(dictionaries.builder==null && dictionaries.model==null){
- $scope.isReuse=2;
- }else if (dictionaries.model!=null){
- $scope.isReuse=3;
+ $scope.isReuse = ReuseEnum.SELF;
+ } else if (dictionaries.model!=null){
+ $scope.isReuse = ReuseEnum.DOMAIN;
}
else{
- $scope.isReuse=1;
+ $scope.isReuse = ReuseEnum.BUILD;
}
}
else{
@@ -235,7 +241,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
$scope.initUpdateDictionariesStatus();
$scope.nextDictionariesInit();
$scope.addNew = !$scope.addNew;
- $scope.isReuse = 1;
+ $scope.isReuse = ReuseEnum.BUILD;
return true;
};
@@ -267,7 +273,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
$scope.clearNewDictionaries = function (){
$scope.newDictionaries = null;
- $scope.isReuse=1;
+ $scope.isReuse = ReuseEnum.BUILD;
$scope.initUpdateDictionariesStatus();
$scope.nextDictionariesInit();
$scope.addNew=!$scope.addNew;
@@ -280,11 +286,11 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
$scope.newDictionaries.model=null;
$scope.newDictionaries.cube=null;
if(type == 'domain'){
- $scope.isReuse=3;
+ $scope.isReuse = ReuseEnum.DOMAIN;
}else if (type == 'builder'){
- $scope.isReuse=1;
+ $scope.isReuse = ReuseEnum.BUILD;
}else if (type == 'reuse'){
- $scope.isReuse=2;
+ $scope.isReuse = ReuseEnum.SELF;
}
}
[kylin] 03/06: KYLIN-4337 Global domain dict for Spark build engine
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit eaa1cdbb5a243798e7a59ea081434b1e0ffcedf9
Author: wangxiaojing <wa...@didichuxing.com>
AuthorDate: Mon Jan 13 20:51:15 2020 +0800
KYLIN-4337 Global domain dict for Spark build engine
---
.../main/java/org/apache/kylin/engine/spark/SparkExecutable.java | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index a95d3d6..df13969 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -44,6 +44,7 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
+import org.apache.kylin.cube.model.CubeDescTiretreeGlobalDomainDictUtil;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -506,6 +507,9 @@ public class SparkExecutable extends AbstractExecutable {
// cube statistics is not available for new segment
dumpList.add(segment.getStatisticsResourcePath());
}
+ //tiretree global domain dic
+ CubeDescTiretreeGlobalDomainDictUtil.cuboidJob(segment.getCubeDesc(), dumpList);
+
JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segment.getConfig(),
this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt()));
}
@@ -520,6 +524,8 @@ public class SparkExecutable extends AbstractExecutable {
// cube statistics is not available for new segment
dumpList.add(segment.getStatisticsResourcePath());
}
+ //tiretree global domain dic
+ CubeDescTiretreeGlobalDomainDictUtil.cuboidJob(segment.getCubeDesc(), dumpList);
}
JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segments.get(0).getConfig(),
this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt()));