You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/05/01 16:06:18 UTC
[11/12] incubator-kylin git commit: Fix null value decode issue in
mergecuboidmapper
Fix null value decode issue in mergecuboidmapper
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/60c68c1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/60c68c1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/60c68c1d
Branch: refs/heads/0.7.1
Commit: 60c68c1d3f773ca7eca4f4deb639f4e97c963e82
Parents: 7166c0a
Author: honma <ho...@ebay.com>
Authored: Mon Apr 27 17:21:14 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Apr 29 14:01:33 2015 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/dict/Dictionary.java | 6 ++++--
.../java/org/apache/kylin/dict/DateStrDictionaryTest.java | 2 +-
.../java/org/apache/kylin/dict/TrieDictionaryTest.java | 2 +-
.../apache/kylin/job/hadoop/cube/MergeCuboidMapper.java | 10 ++++++++--
4 files changed, 14 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/60c68c1d/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
index ed6bcb9..6a06ed2 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
@@ -145,13 +145,15 @@ abstract public class Dictionary<T> implements Writable {
* A lower level API, get byte values from ID, return the number of bytes
* written. Bypassing the cache layer, this could be significantly slower
* than getIdFromValue(T value).
- *
+ *
+ * @return size of value bytes, 0 if empty string, -1 if null
+ *
* @throws IllegalArgumentException
* if ID is not found in dictionary
*/
final public int getValueBytesFromId(int id, byte[] returnValue, int offset) {
if (isNullId(id))
- return 0;
+ return -1;
else
return getValueBytesFromIdImpl(id, returnValue, offset);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/60c68c1d/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
index f393ead..81db5df 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
@@ -64,7 +64,7 @@ public class DateStrDictionaryTest {
int nullId = dict.getIdFromValue(null);
assertNull(dict.getValueFromId(nullId));
int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), 0);
+ assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
assertEquals(nullId, nullId2);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/60c68c1d/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
index fb69a8c..6a8e19d 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
@@ -274,7 +274,7 @@ public class TrieDictionaryTest {
int nullId = dict.getIdFromValue(null);
assertNull(dict.getValueFromId(nullId));
int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), 0);
+ assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
assertEquals(nullId, nullId2);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/60c68c1d/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java
index 431f2b7..d0a0d19 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapper.java
@@ -162,10 +162,16 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
}
int idInSourceDict = BytesUtil.readUnsigned(splittedByteses[i + 1].value, 0, splittedByteses[i + 1].length);
+ int idInMergedDict;
+
int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBuf, bufOffset);
- int idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBuf, bufOffset, size);
- BytesUtil.writeUnsigned(idInMergedDict, newKeyBuf, bufOffset, mergedDict.getSizeOfId());
+ if (size < 0) {
+ idInMergedDict = mergedDict.nullId();
+ } else {
+ idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBuf, bufOffset, size);
+ }
+ BytesUtil.writeUnsigned(idInMergedDict, newKeyBuf, bufOffset, mergedDict.getSizeOfId());
bufOffset += mergedDict.getSizeOfId();
} else {
// keep as it is