You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/05/18 16:08:39 UTC
incubator-kylin git commit: KYLIN-760 Fix the error in cube build
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 83cedc628 -> fff4df99b
KYLIN-760 Fix the error in cube build
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/fff4df99
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/fff4df99
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/fff4df99
Branch: refs/heads/0.8.0
Commit: fff4df99b77fb518b02f9ab5c841fe72c74357c4
Parents: 83cedc6
Author: shaofengshi <sh...@apache.org>
Authored: Mon May 18 22:02:31 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon May 18 22:07:56 2015 +0800
----------------------------------------------------------------------
.../job/hadoop/cubev2/InMemCubeBuilder.java | 86 ++++++++++++--------
.../kylin/storage/cube/CubeCodeSystem.java | 2 +-
.../storage/cube/CubeHBaseReadonlyStore.java | 4 +-
3 files changed, 53 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fff4df99/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
index 413b22c..392ac23 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
@@ -33,17 +33,8 @@
*/
package org.apache.kylin.job.hadoop.cubev2;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.util.StringUtils;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
@@ -56,26 +47,23 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.cube.model.HBaseColumnDesc;
+import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.cube.CubeGridTable;
-import org.apache.kylin.storage.gridtable.GTAggregateScanner;
-import org.apache.kylin.storage.gridtable.GTBuilder;
-import org.apache.kylin.storage.gridtable.GTComboStore;
-import org.apache.kylin.storage.gridtable.GTInfo;
-import org.apache.kylin.storage.gridtable.GTRecord;
-import org.apache.kylin.storage.gridtable.GTScanRequest;
-import org.apache.kylin.storage.gridtable.GridTable;
-import org.apache.kylin.storage.gridtable.IGTScanner;
+import org.apache.kylin.storage.gridtable.*;
import org.apache.kylin.storage.util.SizeOfUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.BlockingQueue;
/**
*/
@@ -95,6 +83,9 @@ public class InMemCubeBuilder implements Runnable {
private String[] metricsAggrFuncs = null;
private Map<Integer, Integer> dependentMeasures = null; // key: index of Measure which depends on another measure; value: index of Measure which is depended on;
public static final LongWritable ONE = new LongWritable(1l);
+ private int[] hbaseMeasureRefIndex;
+ private MeasureDesc[] measureDescs;
+ private int measureCount;
protected IGTRecordWriter gtRecordWriter;
@@ -120,24 +111,43 @@ public class InMemCubeBuilder implements Runnable {
Map<String, Integer> measureIndexMap = Maps.newHashMap();
List<String> metricsAggrFuncsList = Lists.newArrayList();
- final int measureCount = desc.getMeasures().size();
+ measureCount = desc.getMeasures().size();
+
+ List<MeasureDesc> measureDescsList = Lists.newArrayList();
+ hbaseMeasureRefIndex = new int[measureCount];
+ int measureRef = 0;
+ for (HBaseColumnFamilyDesc familyDesc : desc.getHbaseMapping().getColumnFamily()) {
+ for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
+ for (MeasureDesc measure : hbaseColDesc.getMeasures()) {
+ for (int j = 0; j < measureCount; j++) {
+ if (desc.getMeasures().get(j).equals(measure)) {
+ measureDescsList.add(measure);
+ hbaseMeasureRefIndex[measureRef] = j;
+ break;
+ }
+ }
+ measureRef++;
+ }
+ }
+ }
+
for (int i = 0; i < measureCount; i++) {
- MeasureDesc measureDesc = desc.getMeasures().get(i);
+ MeasureDesc measureDesc = measureDescsList.get(i);
metricsAggrFuncsList.add(measureDesc.getFunction().getExpression());
-
- measureIndexMap.put(desc.getMeasures().get(i).getName(), i);
+ measureIndexMap.put(measureDesc.getName(), i);
}
this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
this.dependentMeasures = Maps.newHashMap();
for (int i = 0; i < measureCount; i++) {
- String depMsrRef = desc.getMeasures().get(i).getDependentMeasureRef();
+ String depMsrRef = measureDescsList.get(i).getDependentMeasureRef();
if (depMsrRef != null) {
int index = measureIndexMap.get(depMsrRef);
dependentMeasures.put(i, index);
}
}
+ measureDescs = desc.getMeasures().toArray(new MeasureDesc[measureCount]);
}
@@ -191,7 +201,7 @@ public class InMemCubeBuilder implements Runnable {
try {
BitSet dependentMetrics = new BitSet(allNeededColumns.cardinality());
for (Integer i : dependentMeasures.keySet()) {
- dependentMetrics.set((allNeededColumns.cardinality() - desc.getMeasures().size() + dependentMeasures.get(i)));
+ dependentMetrics.set((allNeededColumns.cardinality() - measureCount + dependentMeasures.get(i)));
}
Object[] hllObjects = new Object[dependentMeasures.keySet().size()];
@@ -207,13 +217,13 @@ public class InMemCubeBuilder implements Runnable {
for (Integer i : dependentMeasures.keySet()) {
for (int index = 0, c = dependentMetrics.nextSetBit(0); c >= 0; index++, c = dependentMetrics.nextSetBit(c + 1)) {
- if (c == allNeededColumns.cardinality() - desc.getMeasures().size() + dependentMeasures.get(i)) {
+ if (c == allNeededColumns.cardinality() - measureCount + dependentMeasures.get(i)) {
assert hllObjects[index] instanceof HyperLogLogPlusCounter; // currently only HLL is allowed
byteBuffer.clear();
BytesUtil.writeVLong(((HyperLogLogPlusCounter) hllObjects[index]).getCountEstimate(), byteBuffer);
byteArray.set(byteBuffer.array(), 0, byteBuffer.position());
- newRecord.set(allNeededColumns.cardinality() - desc.getMeasures().size() + i, byteArray);
+ newRecord.set(allNeededColumns.cardinality() - measureCount + i, byteArray);
}
}
@@ -234,7 +244,7 @@ public class InMemCubeBuilder implements Runnable {
BitSet dimension = new BitSet();
dimension.set(0, bitSet.cardinality());
BitSet metrics = new BitSet();
- metrics.set(bitSet.cardinality(), bitSet.cardinality() + this.desc.getMeasures().size());
+ metrics.set(bitSet.cardinality(), bitSet.cardinality() + this.measureCount);
return new Pair<BitSet, BitSet>(dimension, metrics);
}
@@ -251,10 +261,13 @@ public class InMemCubeBuilder implements Runnable {
private Object[] buildValue(List<String> row) {
- Object[] values = new Object[desc.getMeasures().size()];
+ Object[] values = new Object[measureCount];
MeasureDesc measureDesc = null;
- for (int i = 0, n = desc.getMeasures().size(); i < n; i++) {
- measureDesc = desc.getMeasures().get(i);
+
+ for (int position = 0; position < hbaseMeasureRefIndex.length; position++) {
+ int i = hbaseMeasureRefIndex[position];
+ measureDesc = measureDescs[i];
+
Object value = null;
int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[i];
FunctionDesc function = desc.getMeasures().get(i).getFunction();
@@ -281,7 +294,7 @@ public class InMemCubeBuilder implements Runnable {
}
value = measureCodec.getSerializer(i).valueOf(result);
}
- values[i] = value;
+ values[position] = value;
}
return values;
}
@@ -432,7 +445,7 @@ public class InMemCubeBuilder implements Runnable {
private void createNDCuboidGT(SimpleGridTableTree parentNode, long parentCuboidId, long cuboidId) throws IOException {
long startTime = System.currentTimeMillis();
-
+
GTComboStore parentStore = (GTComboStore) parentNode.data.getStore();
if (parentStore.memoryUsage() <= 0) {
long t = System.currentTimeMillis();
@@ -502,7 +515,8 @@ public class InMemCubeBuilder implements Runnable {
}
}
- private static class SimpleGridTableTree extends TreeNode<GridTable> {}
+ private static class SimpleGridTableTree extends TreeNode<GridTable> {
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fff4df99/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
index 786f154..881e7a1 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
@@ -6,7 +6,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.kv.RowConstants;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fff4df99/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
index 8e5dab3..cb67fc5 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
@@ -12,8 +12,8 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.cube.CubeSegment;