You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/21 01:27:10 UTC
[05/50] [abbrv] incubator-kylin git commit: refactor
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2fce2ba7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2fce2ba7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2fce2ba7
Branch: refs/heads/streaming-localdict
Commit: 2fce2ba76d430cf3892373ae82b456b8c254d40c
Parents: bd9fc31
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Mar 18 16:08:34 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Mar 18 16:08:34 2015 +0800
----------------------------------------------------------------------
.../invertedindex/index/TableRecordInfo.java | 26 ----------------
.../invertedindex/model/IIKeyValueCodec.java | 32 +++++++++++++++++++-
2 files changed, 31 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2fce2ba7/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index 7d87f99..3136ebb 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -108,33 +108,7 @@ public class TableRecordInfo {
return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, dataTypes);
}
- public static TableRecordInfoDigest createDigest(int nColumns, boolean[] isMetric, String[] dataTypes, Map<Integer, Dictionary<?>> dictionaryMap) {
- int[] dictMaxIds = new int[nColumns];
- int[] lengths = new int[nColumns];
- for (int i = 0; i < nColumns; ++i) {
- if (isMetric[i]) {
- final FixedLenMeasureCodec<?> fixedLenMeasureCodec = FixedLenMeasureCodec.get(DataType.getInstance(dataTypes[i]));
- lengths[i] = fixedLenMeasureCodec.getLength();
- } else {
- final Dictionary<?> dictionary = dictionaryMap.get(i);
- if (dictionary != null) {
- lengths[i] = dictionary.getSizeOfId();
- dictMaxIds[i] = dictionary.getMaxId();
- }
- }
- }
- // offsets
- int pos = 0;
- int[] offsets = new int[nColumns];
- for (int i = 0; i < nColumns; i++) {
- offsets[i] = pos;
- pos += lengths[i];
- }
-
- int byteFormLen = pos;
- return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, dataTypes);
- }
public TableRecord createTableRecord() {
return new TableRecord(digest.createTableRecordBytes(), this);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2fce2ba7/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index d15f56b..ee26181 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -26,6 +26,8 @@ import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.index.*;
+import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
+import org.apache.kylin.metadata.model.DataType;
import java.io.*;
import java.util.*;
@@ -122,6 +124,34 @@ public class IIKeyValueCodec implements KeyValueCodec {
// return new Decoder(kvs, incompleteDigest);
}
+ private static TableRecordInfoDigest createDigest(int nColumns, boolean[] isMetric, String[] dataTypes, Map<Integer, Dictionary<?>> dictionaryMap) {
+ int[] dictMaxIds = new int[nColumns];
+ int[] lengths = new int[nColumns];
+ for (int i = 0; i < nColumns; ++i) {
+ if (isMetric[i]) {
+ final FixedLenMeasureCodec<?> fixedLenMeasureCodec = FixedLenMeasureCodec.get(DataType.getInstance(dataTypes[i]));
+ lengths[i] = fixedLenMeasureCodec.getLength();
+ } else {
+ final Dictionary<?> dictionary = dictionaryMap.get(i);
+ if (dictionary != null) {
+ lengths[i] = dictionary.getSizeOfId();
+ dictMaxIds[i] = dictionary.getMaxId();
+ }
+ }
+ }
+ // offsets
+ int pos = 0;
+ int[] offsets = new int[nColumns];
+ for (int i = 0; i < nColumns; i++) {
+ offsets[i] = pos;
+ pos += lengths[i];
+ }
+
+ int byteFormLen = pos;
+
+ return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, dataTypes);
+ }
+
private static class IIRowDecoder implements Iterable<Slice> {
private final TableRecordInfoDigest incompleteDigest;
@@ -184,7 +214,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
}
Preconditions.checkArgument(columns == incompleteDigest.getColumnCount(), "column count is " + columns + " should be equals to incompleteDigest.getColumnCount() " + incompleteDigest.getColumnCount());
- TableRecordInfoDigest digest = TableRecordInfo.createDigest(columns, incompleteDigest.getIsMetric(), incompleteDigest.getMetricDataTypes(), localDictionaries);
+ TableRecordInfoDigest digest = createDigest(columns, incompleteDigest.getIsMetric(), incompleteDigest.getMetricDataTypes(), localDictionaries);
Slice slice = new Slice(digest, curShard, curTimestamp, valueContainers);
slice.setLocalDictionaries(localDictionaries);
return slice;