You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/10/12 07:18:50 UTC
incubator-kylin git commit: KYLIN-1066 fix “Only 1 reducer is started in the "Build cube" step of MR_Engine_V2”
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging e6eb9b363 -> 17e637a09
KYLIN-1066 fix “Only 1 reducer is started in the "Build cube" step of MR_Engine_V2”
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/17e637a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/17e637a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/17e637a0
Branch: refs/heads/2.x-staging
Commit: 17e637a09a588c9fda9e8779c604e8453ad43b59
Parents: e6eb9b3
Author: shaofengshi <sh...@apache.org>
Authored: Mon Oct 12 13:17:52 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Oct 12 13:18:22 2015 +0800
----------------------------------------------------------------------
.../hbase/steps/HBaseMROutput2Transition.java | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/17e637a0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index 00cecf1..6ccb60b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -37,14 +38,14 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.kv.RowConstants;
@@ -360,6 +361,18 @@ public class HBaseMROutput2Transition implements IMROutput2 {
public void configureOutput(Class<? extends Reducer> reducer, String jobFlowId, Job job) throws IOException {
job.setReducerClass(reducer);
+ // estimate the reducer number
+ String htableName = seg.getStorageLocationIdentifier();
+ Configuration conf = HBaseConfiguration.create(job.getConfiguration());
+ HTable htable = new HTable(conf, htableName);
+
+ int regions = htable.getStartKeys().length + 1;
+ int reducerNum = regions * 10;
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ reducerNum = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), reducerNum);
+
+ job.setNumReduceTasks(reducerNum);
+
// the cuboid file and KV class must be compatible with 0.7 version for smooth upgrade
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setOutputKeyClass(Text.class);