You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/05/25 04:35:41 UTC
incubator-kylin git commit: KYLIN-775 Simplify the merge cube segment
codes
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 7d8ae6e38 -> cd7b91869
KYLIN-775 Simplify the merge cube segment codes
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cd7b9186
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cd7b9186
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cd7b9186
Branch: refs/heads/0.8.0
Commit: cd7b918698d558e59992d0319804e9e43f8d0d1a
Parents: 7d8ae6e
Author: shaofengshi <sh...@apache.org>
Authored: Mon May 25 10:33:12 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon May 25 10:35:00 2015 +0800
----------------------------------------------------------------------
.../job/hadoop/cubev2/InMemCuboidReducer.java | 21 ++++-------
.../cubev2/MergeCuboidFromHBaseMapper.java | 39 +++++++-------------
2 files changed, 22 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cd7b9186/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidReducer.java
index cd4c877..adf351b 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCuboidReducer.java
@@ -47,19 +47,6 @@ public class InMemCuboidReducer extends KylinReducer<ImmutableBytesWritable, Tex
CubeDesc cubeDesc = CubeManager.getInstance(config).getCube(cubeName).getDescriptor();
List<MeasureDesc> measuresDescs = Lists.newArrayList();
- for (HBaseColumnFamilyDesc familyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
- for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
- for (MeasureDesc measure : hbaseColDesc.getMeasures()) {
- measuresDescs.add(measure);
- }
- }
- }
-
- codec = new MeasureCodec(measuresDescs);
- aggs = new MeasureAggregators(measuresDescs);
-
- input = new Object[measuresDescs.size()];
- result = new Object[measuresDescs.size()];
keyValueCreators = Lists.newArrayList();
@@ -68,9 +55,17 @@ public class InMemCuboidReducer extends KylinReducer<ImmutableBytesWritable, Tex
for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
keyValueCreators.add(new InMemKeyValueCreator(colDesc, startPosition));
startPosition += colDesc.getMeasures().length;
+ for (MeasureDesc measure : colDesc.getMeasures()) {
+ measuresDescs.add(measure);
+ }
}
}
+ codec = new MeasureCodec(measuresDescs);
+ aggs = new MeasureAggregators(measuresDescs);
+
+ input = new Object[measuresDescs.size()];
+ result = new Object[measuresDescs.size()];
nColumns = keyValueCreators.size();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cd7b9186/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
index 59fa03d..ef59d9b 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeCuboidFromHBaseMapper.java
@@ -86,7 +86,6 @@ public class MergeCuboidFromHBaseMapper extends TableMapper<ImmutableBytesWritab
private boolean simpleFullCopy = false;
private Object[] result;
private MeasureCodec codec;
- private int[][] hbaseColumnToMeasureMapping;
private Boolean checkNeedMerging(TblColRef col) throws IOException {
Boolean ret = dictsNeedMerging.get(col);
@@ -143,35 +142,25 @@ public class MergeCuboidFromHBaseMapper extends TableMapper<ImmutableBytesWritab
this.rowKeySplitter = new RowKeySplitter(sourceCubeSegment, 65, 255);
List<RowValueDecoder> valueDecoderList = Lists.newArrayList();
- List<KeyValueCreator> keyValueCreators = Lists.newArrayList();
+ List<InMemKeyValueCreator> keyValueCreators = Lists.newArrayList();
+ List<MeasureDesc> measuresDescs = Lists.newArrayList();
+ int startPosition = 0;
for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
valueDecoderList.add(new RowValueDecoder(colDesc));
- keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
+ keyValueCreators.add(new InMemKeyValueCreator(colDesc, startPosition));
+ startPosition += colDesc.getMeasures().length;
+ for (MeasureDesc measure : colDesc.getMeasures()) {
+ measuresDescs.add(measure);
+ }
}
}
rowValueDecoders = valueDecoderList.toArray(new RowValueDecoder[valueDecoderList.size()]);
- simpleFullCopy = (keyValueCreators.size() == 1 && keyValueCreators.get(0).isFullCopy);
- result = new Object[cubeDesc.getMeasures().size()];
- codec = new MeasureCodec(cubeDesc.getMeasures());
-
- hbaseColumnToMeasureMapping = new int[rowValueDecoders.length][];
-
- for (int i = 0; i < rowValueDecoders.length; i++) {
- hbaseColumnToMeasureMapping[i] = new int[rowValueDecoders[i].getMeasures().length];
- for (int j = 0; j < rowValueDecoders[i].getMeasures().length; j++) {
- int positionInCubeMeasures = 0;
- for (MeasureDesc m : cubeDesc.getMeasures()) {
- if (m.equals(rowValueDecoders[i].getMeasures()[j])) {
- hbaseColumnToMeasureMapping[i][j] = positionInCubeMeasures;
- break;
- }
- positionInCubeMeasures++;
- }
- }
- }
+ simpleFullCopy = (keyValueCreators.size() == 1);
+ result = new Object[measuresDescs.size()];
+ codec = new MeasureCodec(measuresDescs);
}
@Override
@@ -222,18 +211,18 @@ public class MergeCuboidFromHBaseMapper extends TableMapper<ImmutableBytesWritab
valueBuf.clear();
if (simpleFullCopy) {
- // simple case, should only 1 hbase column and the bytes sequence is same
+ // simple case, should only 1 hbase column
for (Cell cell : value.rawCells()) {
valueBuf.put(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
}
} else {
- // complex case, need re-arrange the bytes sequence
+ int position = 0;
for (int i = 0; i < rowValueDecoders.length; i++) {
rowValueDecoders[i].decode(value, false);
Object[] measureValues = rowValueDecoders[i].getValues();
for (int j = 0; j < measureValues.length; j++) {
- result[hbaseColumnToMeasureMapping[i][j]] = measureValues[j];
+ result[position++] = measureValues[j];
}
}
codec.encode(result, valueBuf);