You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/01/05 03:35:48 UTC
kylin git commit: KYLIN-2351 more enforce Path schema
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2351 aab7178c4 -> 2ac1743fc
KYLIN-2351 more enforce Path schema
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ac1743f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ac1743f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ac1743f
Branch: refs/heads/KYLIN-2351
Commit: 2ac1743fc2c42cd0346576e8dcd6674990fc6bc6
Parents: aab7178
Author: Li Yang <li...@apache.org>
Authored: Thu Jan 5 11:35:44 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Jan 5 11:35:44 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java | 2 +-
.../kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2ac1743f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
index 88f6ba2..af86181 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
@@ -123,7 +123,7 @@ public class MergeStatisticsStep extends AbstractExecutable {
averageSamplingPercentage = averageSamplingPercentage / CubingExecutableUtil.getMergingSegmentIds(this.getParams()).size();
CubeStatsWriter.writeCuboidStatistics(conf, new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams())), cuboidHLLMap, averageSamplingPercentage);
Path statisticsFilePath = new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams()), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
- FileSystem fs = statisticsFilePath.getFileSystem(conf);
+ FileSystem fs = HadoopUtil.getFileSystem(statisticsFilePath, conf);
FSDataInputStream is = fs.open(statisticsFilePath);
try {
// put the statistics to metadata store
http://git-wip-us.apache.org/repos/asf/kylin/blob/2ac1743f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
index fca91a6..ee8da6b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.KylinMapper;
/**
@@ -89,7 +90,7 @@ public class RowKeyDistributionCheckerMapper extends KylinMapper<Text, Text, Tex
List<byte[]> rowkeyList = new ArrayList<byte[]>();
SequenceFile.Reader reader = null;
try {
- reader = new SequenceFile.Reader(path.getFileSystem(conf), path, conf);
+ reader = new SequenceFile.Reader(HadoopUtil.getFileSystem(path, conf), path, conf);
Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
while (reader.next(key, value)) {