You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/01/23 09:32:01 UTC
[14/50] [abbrv] incubator-kylin git commit: Merge branch 'staging'
into inverted-index
Merge branch 'staging' into inverted-index
Conflicts:
job/src/main/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducer.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8437556e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8437556e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8437556e
Branch: refs/heads/inverted-index
Commit: 8437556e7a77289d6164ac0ad7a2d91815055bdc
Parents: 6cda6b2 46fd452
Author: liyang@apache.org <ya...@D-SHC-00801746.corp.ebay.com>
Authored: Thu Jan 22 08:41:26 2015 +0000
Committer: liyang@apache.org <ya...@D-SHC-00801746.corp.ebay.com>
Committed: Thu Jan 22 08:41:26 2015 +0000
----------------------------------------------------------------------
.../cube/RangeKeyDistributionReducer.java | 68 +++++++++++---------
.../job/hadoop/hbase/CreateHTableJob.java | 26 +++++---
2 files changed, 53 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8437556e/job/src/main/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducer.java
----------------------------------------------------------------------
diff --cc job/src/main/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducer.java
index aa04ef6,e7f22cf..f08a79c
--- a/job/src/main/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducer.java
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducer.java
@@@ -21,12 -23,10 +23,12 @@@ import java.util.List
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.kylinolap.common.mr.KylinReducer;
- import com.kylinolap.cube.model.CubeDesc.RealizationCapacity;
++import com.kylinolap.cube.model.v1.CubeDesc.CubeCapacity;
import com.kylinolap.job.constant.BatchConstants;
-import com.kylinolap.metadata.model.cube.CubeDesc.CubeCapacity;
/**
* @author ysong1
@@@ -34,38 -34,35 +36,39 @@@
*/
public class RangeKeyDistributionReducer extends KylinReducer<Text, LongWritable, Text, LongWritable> {
- public static final long TEN_GIGA_BYTES = 10L * 1024L * 1024L * 1024L;
- public static final long TWENTY_GIGA_BYTES = 20L * 1024L * 1024L * 1024L;
- public static final long HUNDRED_GIGA_BYTES = 100L * 1024L * 1024L * 1024L;
+ public static final long ONE_GIGA_BYTES = 1024L * 1024L * 1024L;
+ public static final int SMALL_CUT = 10; // 10 GB per region
+ public static final int MEDIUM_CUT = 20; // 20 GB per region
+ public static final int LARGE_CUT = 100; // 100 GB per region
+
+ public static final int MAX_REGION = 200;
+ private static final Logger logger = LoggerFactory.getLogger(RangeKeyDistributionReducer.class);
+
private LongWritable outputValue = new LongWritable(0);
+ private int cut;
private long bytesRead = 0;
- private Text lastKey;
-
- private RealizationCapacity realizationCapacity;
- private long cut;
+ private List<Text> gbPoints = new ArrayList<Text>();
@Override
protected void setup(Context context) throws IOException {
super.publishConfiguration(context.getConfiguration());
-
- realizationCapacity = RealizationCapacity.valueOf(context.getConfiguration().get(BatchConstants.CUBE_CAPACITY));
- switch (realizationCapacity) {
+
+ CubeCapacity cubeCapacity = CubeCapacity.valueOf(context.getConfiguration().get(BatchConstants.CUBE_CAPACITY));
+ switch (cubeCapacity) {
case SMALL:
- cut = TEN_GIGA_BYTES;
+ cut = SMALL_CUT;
break;
case MEDIUM:
- cut = TWENTY_GIGA_BYTES;
+ cut = MEDIUM_CUT;
break;
case LARGE:
- cut = HUNDRED_GIGA_BYTES;
+ cut = LARGE_CUT;
break;
}
+
+ logger.info("Chosen cut for htable is " + cut);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8437556e/job/src/main/java/com/kylinolap/job/hadoop/hbase/CreateHTableJob.java
----------------------------------------------------------------------