You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2017/02/04 06:16:56 UTC
[34/39] kylin git commit: KYLIN-2414 Distinguish UHC columns from
normal columns in KYLIN-2217
KYLIN-2414 Distinguish UHC columns from normal columns in KYLIN-2217
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6a9382b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6a9382b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6a9382b
Branch: refs/heads/spark-it
Commit: e6a9382b8d357e5bb13be2f678ebabb8bad75dc7
Parents: 5eae37e
Author: xiefan46 <95...@qq.com>
Authored: Tue Jan 24 11:14:40 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 26 09:56:54 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 3 ---
.../kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 6 ++++++
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a9382b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b1acbbf..b25bcc0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -741,9 +741,6 @@ abstract public class KylinConfigBase implements Serializable {
}
public boolean isReducerLocalBuildDict() {
- if (getUHCReducerCount() != 1) {
- return false;
- }
return Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", "true"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a9382b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8c56bdf..cf94b30 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -132,6 +132,12 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
if (cubeDesc.getDictionaryBuilderClass(col) != null) { // only works with default dictionary builder
isReducerLocalBuildDict = false;
}
+ if(config.getUHCReducerCount() > 1) {
+ int[] uhcIndex = CubeManager.getInstance(config).getUHCIndex(cubeDesc);
+ int colIndex = reducerIdToColumnIndex.get(taskId);
+ if (uhcIndex[colIndex] == 1)
+ isReducerLocalBuildDict = false; //for UHC columns, this feature should be disabled
+ }
if (isReducerLocalBuildDict) {
builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
builder.init(null, 0);