You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by su...@apache.org on 2016/08/13 04:57:15 UTC
kylin git commit: KYLIN-1934 'Value not exist' During Cube Merging
Caused by Empty Dict
Repository: kylin
Updated Branches:
refs/heads/master 8ecb7d455 -> 47d2d15c6
KYLIN-1934 'Value not exist' During Cube Merging Caused by Empty Dict
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/47d2d15c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/47d2d15c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/47d2d15c
Branch: refs/heads/master
Commit: 47d2d15c6643c56e1fa6c65308d82ef1baf3b323
Parents: 8ecb7d4
Author: sunyerui <su...@gmail.com>
Authored: Tue Aug 2 23:27:57 2016 +0800
Committer: sunyerui <su...@gmail.com>
Committed: Sat Aug 13 12:56:26 2016 +0800
----------------------------------------------------------------------
.../dict/MultipleDictionaryValueEnumerator.java | 22 +--
.../MultipleDictionaryValueEnumeratorTest.java | 149 +++++++++++++++++++
2 files changed, 161 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/47d2d15c/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
index 92ff951..ac38319 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
@@ -55,19 +55,21 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer
@Override
public boolean moveNext() throws IOException {
- if (curDictIndex < dictionaryList.size() && curKey <= curDict.getMaxId()) {
- byte[] buffer = new byte[curDict.getSizeOfValue()];
- int size = curDict.getValueBytesFromId(curKey, buffer, 0);
- curValue = Bytes.copy(buffer, 0, size);
+ while (curDictIndex < dictionaryList.size()) {
+ if (curKey <= curDict.getMaxId()) {
+ byte[] buffer = new byte[curDict.getSizeOfValue()];
+ int size = curDict.getValueBytesFromId(curKey, buffer, 0);
+ curValue = Bytes.copy(buffer, 0, size);
+ curKey ++;
- if (++curKey > curDict.getMaxId()) {
- if (++curDictIndex < dictionaryList.size()) {
- curDict = dictionaryList.get(curDictIndex);
- curKey = curDict.getMinId();
- }
+ return true;
}
- return true;
+ // move to next dict if exists
+ if (++curDictIndex < dictionaryList.size()) {
+ curDict = dictionaryList.get(curDictIndex);
+ curKey = curDict.getMinId();
+ }
}
curValue = null;
return false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/47d2d15c/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
new file mode 100644
index 0000000..6e0f88a
--- /dev/null
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
@@ -0,0 +1,149 @@
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Dictionary;
+import org.junit.Test;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Created by sunyerui on 16/8/2.
+ */
+public class MultipleDictionaryValueEnumeratorTest {
+
+ private static DictionaryInfo createDictInfo(int[] values) {
+ MockDictionary mockDict = new MockDictionary();
+ mockDict.values = values;
+ DictionaryInfo info = new DictionaryInfo();
+ info.setDictionaryObject(mockDict);
+ return info;
+ }
+
+ private static Integer[] enumerateDictInfoList(List<DictionaryInfo> dictionaryInfoList) throws IOException {
+ MultipleDictionaryValueEnumerator enumerator = new MultipleDictionaryValueEnumerator(dictionaryInfoList);
+ List<Integer> values = new ArrayList<>();
+ while (enumerator.moveNext()) {
+ values.add(Bytes.toInt(enumerator.current()));
+ }
+ return values.toArray(new Integer[0]);
+ }
+
+ @Test
+ public void testNormalDicts() throws IOException {
+ List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(2);
+ dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2}));
+ dictionaryInfoList.add(createDictInfo(new int[]{4, 5, 6}));
+
+ Integer[] values = enumerateDictInfoList(dictionaryInfoList);
+ assertEquals(6, values.length);
+ assertArrayEquals(new Integer[]{0, 1, 2, 4, 5, 6}, values);
+ }
+
+ @Test
+ public void testFirstEmptyDicts() throws IOException {
+ List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(2);
+ dictionaryInfoList.add(createDictInfo(new int[]{}));
+ dictionaryInfoList.add(createDictInfo(new int[]{4, 5, 6}));
+
+ Integer[] values = enumerateDictInfoList(dictionaryInfoList);
+ assertEquals(3, values.length);
+ assertArrayEquals(new Integer[]{4, 5, 6}, values);
+ }
+
+ @Test
+ public void testMiddleEmptyDicts() throws IOException {
+ List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(3);
+ dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2}));
+ dictionaryInfoList.add(createDictInfo(new int[]{}));
+ dictionaryInfoList.add(createDictInfo(new int[]{7, 8, 9}));
+
+ Integer[] values = enumerateDictInfoList(dictionaryInfoList);
+ assertEquals(6, values.length);
+ assertArrayEquals(new Integer[]{0, 1, 2, 7, 8, 9}, values);
+ }
+
+ @Test
+ public void testLastEmptyDicts() throws IOException {
+ List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(3);
+ dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2}));
+ dictionaryInfoList.add(createDictInfo(new int[]{6, 7, 8}));
+ dictionaryInfoList.add(createDictInfo(new int[]{}));
+
+ Integer[] values = enumerateDictInfoList(dictionaryInfoList);
+ assertEquals(6, values.length);
+ assertArrayEquals(new Integer[]{0, 1, 2, 6, 7, 8}, values);
+ }
+
+ public static class MockDictionary extends Dictionary {
+ public int[] values;
+
+ @Override
+ public int getMinId() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxId() {
+ return values.length-1;
+ }
+
+ @Override
+ public int getSizeOfId() {
+ return 4;
+ }
+
+ @Override
+ public int getSizeOfValue() {
+ return 4;
+ }
+
+ @Override
+ protected int getIdFromValueImpl(Object value, int roundingFlag) {
+ return 0;
+ }
+
+ @Override
+ protected Object getValueFromIdImpl(int id) {
+ return null;
+ }
+
+ @Override
+ protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
+ return 0;
+ }
+
+ @Override
+ protected byte[] getValueBytesFromIdImpl(int id) {
+ return null;
+ }
+
+ @Override
+ protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
+ System.arraycopy(Bytes.toBytes(values[id]), 0, returnValue, offset, 4);
+ return 4;
+ }
+
+ @Override
+ public void dump(PrintStream out) {}
+
+ @Override
+ public void write(DataOutput out) throws IOException {}
+
+ @Override
+ public void readFields(DataInput in) throws IOException {}
+
+ @Override
+ public boolean contains(Dictionary another) {
+ return false;
+ }
+ }
+
+}
\ No newline at end of file