You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/02/04 04:44:11 UTC
[42/47] kylin git commit: KYLIN-2414 minor rename on properties
KYLIN-2414 minor rename on properties
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3fff6d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3fff6d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3fff6d1
Branch: refs/heads/KYLIN-2361
Commit: c3fff6d19d355e78461fa7f32d02feabc5bf63c8
Parents: e6a9382
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 26 10:53:19 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 26 10:53:19 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 8 +++++++-
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++--
.../mr/steps/FactDistinctColumnsReducer.java | 19 +++++++++----------
3 files changed, 18 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 43ea17d..1232c47 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -127,9 +127,15 @@ kylin.engine.mr.max-reducer-number=500
kylin.engine.mr.mapper-input-rows=1000000
+# Enable dictionary building in MR reducer
+kylin.engine.mr.build-dict-in-reducer=true
+
+# Number of reducers for fetching UHC column distinct values
+kylin.engine.mr.uhc-reducer-count=1
+
### CUBE | DICTIONARY ###
-# 'auto', 'inmem', 'layer' or 'random' for testing
+# 'auto', 'inmem' or 'layer'
kylin.cube.algorithm=auto
# A smaller threshold prefers layer, a larger threshold prefers in-mem
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b25bcc0..6a88fc4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -740,8 +740,8 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
}
- public boolean isReducerLocalBuildDict() {
- return Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", "true"));
+ public boolean isBuildDictInReducerEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.engine.mr.build-dict-in-reducer", "true"));
}
public String getYarnStatusCheckUrl() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index cf94b30..5b795c2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -78,7 +78,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private int rowCount = 0;
//local build dict
- private boolean isReducerLocalBuildDict;
+ private boolean buildDictInReducer;
private IDictionaryBuilder builder;
private long timeMaxValue = Long.MIN_VALUE;
private long timeMinValue = Long.MAX_VALUE;
@@ -119,30 +119,29 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
isPartitionCol = true;
col = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
if (col == null) {
- logger.info("Do not have partition col. This reducer will keep empty");
+ logger.info("No partition col. This reducer will do nothing");
}
} else {
// normal col
col = columnList.get(reducerIdToColumnIndex.get(taskId));
-
Preconditions.checkNotNull(col);
// local build dict
- isReducerLocalBuildDict = config.isReducerLocalBuildDict();
+ buildDictInReducer = config.isBuildDictInReducerEnabled();
if (cubeDesc.getDictionaryBuilderClass(col) != null) { // only works with default dictionary builder
- isReducerLocalBuildDict = false;
+ buildDictInReducer = false;
}
if(config.getUHCReducerCount() > 1) {
int[] uhcIndex = CubeManager.getInstance(config).getUHCIndex(cubeDesc);
int colIndex = reducerIdToColumnIndex.get(taskId);
if (uhcIndex[colIndex] == 1)
- isReducerLocalBuildDict = false; //for UHC columns, this feature should be disabled
+ buildDictInReducer = false; //for UHC columns, this feature should be disabled
}
- if (isReducerLocalBuildDict) {
+ if (buildDictInReducer) {
builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
builder.init(null, 0);
}
- logger.info("Reducer " + taskId + " handling column " + col + ", isReducerLocalBuildDict=" + isReducerLocalBuildDict);
+ logger.info("Reducer " + taskId + " handling column " + col + ", buildDictInReducer=" + buildDictInReducer);
}
}
@@ -192,7 +191,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
timeMaxValue = Math.max(timeMaxValue, time);
} else {
// normal col
- if (isReducerLocalBuildDict) {
+ if (buildDictInReducer) {
String value = Bytes.toString(key.getBytes(), 1, key.getLength() - 1);
logAFewRows(value);
builder.addValue(value);
@@ -228,7 +227,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
outputPartitionInfo();
} else {
// normal col
- if (isReducerLocalBuildDict) {
+ if (buildDictInReducer) {
Dictionary<String> dict = builder.build();
outputDict(col, dict);
}