You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/30 07:59:13 UTC
kylin git commit: cv++
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2217-2 109772d91 -> cf8eb696a
cv++
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cf8eb696
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cf8eb696
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cf8eb696
Branch: refs/heads/KYLIN-2217-2
Commit: cf8eb696a37635131c933b3b04040c2ecc5b9ee1
Parents: 109772d
Author: Li Yang <li...@apache.org>
Authored: Wed Nov 30 15:59:05 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Nov 30 15:59:05 2016 +0800
----------------------------------------------------------------------
.../engine/mr/steps/FactDistinctColumnsReducer.java | 12 ++++++------
.../engine/mr/steps/UpdateCubeInfoAfterBuildStep.java | 6 +-----
2 files changed, 7 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf8eb696/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 59532e8..af57a07 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -62,6 +62,8 @@ import com.google.common.collect.Maps;
*/
public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableKey, Text, NullWritable, Text> {
+ protected static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
+
private List<TblColRef> columnList;
private String statisticsOutput = null;
private List<Long> baseCuboidRowCountInMappers;
@@ -75,11 +77,9 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private boolean isStatistics = false;
private KylinConfig cubeConfig;
private int uhcReducerCount;
- private Map<Integer, Integer> ReducerIdToColumnIndex = new HashMap<>();
+ private Map<Integer, Integer> reducerIdToColumnIndex = new HashMap<>();
private int taskId;
- protected static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
-
//local build dict
private boolean isReducerLocalBuildDict;
private IDictionaryBuilder builder;
@@ -138,7 +138,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
} else {
// col
isStatistics = false;
- col = columnList.get(ReducerIdToColumnIndex.get(taskId));
+ col = columnList.get(reducerIdToColumnIndex.get(taskId));
colValues = Lists.newLinkedList();
}
@@ -155,10 +155,10 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
int[] uhcIndex = CubeManager.getInstance(config).getUHCIndex(cubeDesc);
int count = 0;
for (int i = 0; i < uhcIndex.length; i++) {
- ReducerIdToColumnIndex.put(count * (uhcReducerCount - 1) + i, i);
+ reducerIdToColumnIndex.put(count * (uhcReducerCount - 1) + i, i);
if (uhcIndex[i] == 1) {
for (int j = 1; j < uhcReducerCount; j++) {
- ReducerIdToColumnIndex.put(count * (uhcReducerCount - 1) + j + i, i);
+ reducerIdToColumnIndex.put(count * (uhcReducerCount - 1) + j + i, i);
}
count++;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf8eb696/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 977196c..d3becfe 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -21,7 +21,6 @@ package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -84,8 +83,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
final String factColumnsInputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
Path colDir = new Path(factColumnsInputPath, partitionCol.getName());
Path outputFile = new Path(colDir, partitionCol.getName() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
- Configuration conf = HadoopUtil.getCurrentConfiguration();
- FileSystem fs = HadoopUtil.getFileSystem(outputFile.getName());
+ FileSystem fs = HadoopUtil.getFileSystem(outputFile.toString());
FSDataInputStream is = null;
long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
try {
@@ -94,8 +92,6 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
long max = is.readLong();
minValue = Math.min(min, minValue);
maxValue = Math.max(max, maxValue);
- } catch (IOException e) {
- throw new IOException(e);
} finally {
IOUtils.closeQuietly(is);
}