You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/05/27 06:34:48 UTC
[23/50] [abbrv] incubator-kylin git commit: Fix null value decode
issue in mergecuboidmapper
Fix null value decode issue in mergecuboidmapper
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b5332e72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b5332e72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b5332e72
Branch: refs/heads/0.8.0
Commit: b5332e72eafb2ce33449edc31289674b1bd93409
Parents: 141614d
Author: honma <ho...@ebay.com>
Authored: Mon Apr 27 17:21:14 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed May 27 12:24:08 2015 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/dict/Dictionary.java | 6 ++++--
.../java/org/apache/kylin/dict/DateStrDictionaryTest.java | 2 +-
.../java/org/apache/kylin/dict/TrieDictionaryTest.java | 2 +-
.../apache/kylin/job/hadoop/cube/MergeCuboidMapper.java | 10 ++++++++--
4 files changed, 14 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b5332e72/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
index 070e9de..cdc42b4 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
@@ -156,13 +156,15 @@ abstract public class Dictionary<T> implements Writable {
* A lower level API, get byte values from ID, return the number of bytes
* written. Bypassing the cache layer, this could be significantly slower
* than getIdFromValue(T value).
- *
+ *
+ * @return size of value bytes, 0 if empty string, -1 if null
+ *
* @throws IllegalArgumentException
* if ID is not found in dictionary
*/
final public int getValueBytesFromId(int id, byte[] returnValue, int offset) {
if (isNullId(id))
- return 0;
+ return -1;
else
return getValueBytesFromIdImpl(id, returnValue, offset);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b5332e72/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
index f393ead..81db5df 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
@@ -64,7 +64,7 @@ public class DateStrDictionaryTest {
int nullId = dict.getIdFromValue(null);
assertNull(dict.getValueFromId(nullId));
int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), 0);
+ assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
assertEquals(nullId, nullId2);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b5332e72/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
index fb69a8c..6a8e19d 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
@@ -274,7 +274,7 @@ public class TrieDictionaryTest {
int nullId = dict.getIdFromValue(null);
assertNull(dict.getValueFromId(nullId));
int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), 0);
+ assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
assertEquals(nullId, nullId2);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b5332e72/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java
index 98ec250..f776029 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java
@@ -162,10 +162,16 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
}
int idInSourceDict = BytesUtil.readUnsigned(splittedByteses[i + 1].value, 0, splittedByteses[i + 1].length);
+ int idInMergedDict;
+
int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBuf, bufOffset);
- int idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBuf, bufOffset, size);
- BytesUtil.writeUnsigned(idInMergedDict, newKeyBuf, bufOffset, mergedDict.getSizeOfId());
+ if (size < 0) {
+ idInMergedDict = mergedDict.nullId();
+ } else {
+ idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBuf, bufOffset, size);
+ }
+ BytesUtil.writeUnsigned(idInMergedDict, newKeyBuf, bufOffset, mergedDict.getSizeOfId());
bufOffset += mergedDict.getSizeOfId();
} else {
// keep as it is