You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/10/08 11:16:07 UTC
[1/2] kylin git commit: KYLIN-2794 MultipleDictionaryValueEnumerator
should output values in sorted order
Repository: kylin
Updated Branches:
refs/heads/misc [created] a82847eaa
KYLIN-2794 MultipleDictionaryValueEnumerator should output values in sorted order
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b86a7346
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b86a7346
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b86a7346
Branch: refs/heads/misc
Commit: b86a7346a1fcc3d5cf0215e83eb5279f1c4872a4
Parents: 0297d9a
Author: yifei.wu <yi...@kyligence.io>
Authored: Tue Sep 26 15:32:11 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Oct 8 18:48:58 2017 +0800
----------------------------------------------------------------------
.../dict/MultipleDictionaryValueEnumerator.java | 41 +++++++++++++-------
.../MultipleDictionaryValueEnumeratorTest.java | 14 +++++++
2 files changed, 42 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b86a7346/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
index c1bc5d5..7bd526b 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
@@ -33,13 +33,16 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer
private int curDictIndex = 0;
private Dictionary<String> curDict;
private int curKey;
+ private List<Integer> curKeys = Lists.newArrayList();
private String curValue = null;
private List<Dictionary<String>> dictionaryList;
public MultipleDictionaryValueEnumerator(List<DictionaryInfo> dictionaryInfoList) {
dictionaryList = Lists.newArrayListWithCapacity(dictionaryInfoList.size());
for (DictionaryInfo dictInfo : dictionaryInfoList) {
+ Dictionary<String> dictionary = (Dictionary<String>) dictInfo.getDictionaryObject();
dictionaryList.add((Dictionary<String>) dictInfo.getDictionaryObject());
+ curKeys.add(dictionary.getMinId());
}
if (!dictionaryList.isEmpty()) {
curDict = dictionaryList.get(0);
@@ -54,22 +57,34 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer
@Override
public boolean moveNext() throws IOException {
- while (curDictIndex < dictionaryList.size()) {
- if (curKey <= curDict.getMaxId()) {
- curValue = curDict.getValueFromId(curKey);
- curKey ++;
-
- return true;
+ //initialize the minValue as the first available value in the dictionary list
+ String minValue = null;
+ int curDictIndex = 0;
+ for (int i = 0; i < dictionaryList.size(); i++) {
+ if (dictionaryList.get(i) == null || curKeys.get(i) >= dictionaryList.get(i).getSize()) {
+ continue;
}
-
- // move to next dict if exists
- if (++curDictIndex < dictionaryList.size()) {
- curDict = dictionaryList.get(curDictIndex);
- curKey = curDict.getMinId();
+ minValue = dictionaryList.get(i).getValueFromId(curKeys.get(i));
+ curDictIndex = i;
+ break;
+ }
+ if (minValue == null) {
+ curValue = null;
+ return false;
+ }
+ // multi-merge dictionary forest
+ for (int i = 0; i < dictionaryList.size(); i++) {
+ if (dictionaryList.get(i) == null || curKeys.get(i) >= dictionaryList.get(i).getSize()) {
+ continue;
+ }
+ if (minValue.compareTo(dictionaryList.get(i).getValueFromId(curKeys.get(i))) > 0) {
+ minValue = dictionaryList.get(i).getValueFromId(curKeys.get(i));
+ curDictIndex = i;
}
}
- curValue = null;
- return false;
+ curValue = minValue;
+ curKeys.set(curDictIndex, curKeys.get(curDictIndex) + 1);
+ return true;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/b86a7346/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
index 2e90bcf..29e9c5d 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
@@ -99,6 +99,20 @@ public class MultipleDictionaryValueEnumeratorTest {
assertArrayEquals(new Integer[]{0, 1, 2, 6, 7, 8}, values);
}
+ @Test
+ public void testUnorderedDicts() throws IOException {
+ List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(3);
+ dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 6}));
+ dictionaryInfoList.add(createDictInfo(new int[]{3, 7, 8}));
+ dictionaryInfoList.add(createDictInfo(new int[]{2, 7, 9}));
+ Integer[] values = enumerateDictInfoList(dictionaryInfoList);
+ assertEquals(9, values.length);
+ assertArrayEquals(new Integer[]{0, 1, 2, 3, 6, 7, 7, 8, 9}, values);
+ }
+
+
+
+
public static class MockDictionary extends Dictionary<String> {
private static final long serialVersionUID = 1L;
[2/2] kylin git commit: KYLIN-2794 code review
Posted by li...@apache.org.
KYLIN-2794 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a82847ea
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a82847ea
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a82847ea
Branch: refs/heads/misc
Commit: a82847eaa24060ab001311aeeb321260c10c3128
Parents: b86a734
Author: Li Yang <li...@apache.org>
Authored: Sun Oct 8 19:15:48 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Oct 8 19:15:48 2017 +0800
----------------------------------------------------------------------
.../dict/MultipleDictionaryValueEnumerator.java | 37 ++++++--------
.../MultipleDictionaryValueEnumeratorTest.java | 54 ++++++++++----------
2 files changed, 41 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a82847ea/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
index 7bd526b..f0d4e34 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
@@ -30,9 +30,6 @@ import com.google.common.collect.Lists;
*/
@SuppressWarnings("rawtypes")
public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumerator {
- private int curDictIndex = 0;
- private Dictionary<String> curDict;
- private int curKey;
private List<Integer> curKeys = Lists.newArrayList();
private String curValue = null;
private List<Dictionary<String>> dictionaryList;
@@ -44,10 +41,6 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer
dictionaryList.add((Dictionary<String>) dictInfo.getDictionaryObject());
curKeys.add(dictionary.getMinId());
}
- if (!dictionaryList.isEmpty()) {
- curDict = dictionaryList.get(0);
- curKey = curDict.getMinId();
- }
}
@Override
@@ -57,31 +50,31 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer
@Override
public boolean moveNext() throws IOException {
- //initialize the minValue as the first available value in the dictionary list
String minValue = null;
int curDictIndex = 0;
+
+ // multi-merge dictionary forest
for (int i = 0; i < dictionaryList.size(); i++) {
- if (dictionaryList.get(i) == null || curKeys.get(i) >= dictionaryList.get(i).getSize()) {
+ Dictionary<String> dict = dictionaryList.get(i);
+ if (dict == null)
+ continue;
+
+ int curKey = curKeys.get(i);
+ if (curKey > dict.getMaxId())
continue;
+
+ String curValue = dict.getValueFromId(curKey);
+ if (minValue == null || minValue.compareTo(curValue) > 0) {
+ minValue = curValue;
+ curDictIndex = i;
}
- minValue = dictionaryList.get(i).getValueFromId(curKeys.get(i));
- curDictIndex = i;
- break;
}
+
if (minValue == null) {
curValue = null;
return false;
}
- // multi-merge dictionary forest
- for (int i = 0; i < dictionaryList.size(); i++) {
- if (dictionaryList.get(i) == null || curKeys.get(i) >= dictionaryList.get(i).getSize()) {
- continue;
- }
- if (minValue.compareTo(dictionaryList.get(i).getValueFromId(curKeys.get(i))) > 0) {
- minValue = dictionaryList.get(i).getValueFromId(curKeys.get(i));
- curDictIndex = i;
- }
- }
+
curValue = minValue;
curKeys.set(curDictIndex, curKeys.get(curDictIndex) + 1);
return true;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a82847ea/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
index 29e9c5d..3496c00 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
@@ -56,66 +56,63 @@ public class MultipleDictionaryValueEnumeratorTest {
@Test
public void testNormalDicts() throws IOException {
List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(2);
- dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2}));
- dictionaryInfoList.add(createDictInfo(new int[]{4, 5, 6}));
+ dictionaryInfoList.add(createDictInfo(new int[] { 0, 1, 2 }));
+ dictionaryInfoList.add(createDictInfo(new int[] { 4, 5, 6 }));
Integer[] values = enumerateDictInfoList(dictionaryInfoList);
assertEquals(6, values.length);
- assertArrayEquals(new Integer[]{0, 1, 2, 4, 5, 6}, values);
+ assertArrayEquals(new Integer[] { 0, 1, 2, 4, 5, 6 }, values);
}
@Test
public void testFirstEmptyDicts() throws IOException {
List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(2);
- dictionaryInfoList.add(createDictInfo(new int[]{}));
- dictionaryInfoList.add(createDictInfo(new int[]{4, 5, 6}));
+ dictionaryInfoList.add(createDictInfo(new int[] {}));
+ dictionaryInfoList.add(createDictInfo(new int[] { 4, 5, 6 }));
Integer[] values = enumerateDictInfoList(dictionaryInfoList);
assertEquals(3, values.length);
- assertArrayEquals(new Integer[]{4, 5, 6}, values);
+ assertArrayEquals(new Integer[] { 4, 5, 6 }, values);
}
@Test
public void testMiddleEmptyDicts() throws IOException {
List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(3);
- dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2}));
- dictionaryInfoList.add(createDictInfo(new int[]{}));
- dictionaryInfoList.add(createDictInfo(new int[]{7, 8, 9}));
+ dictionaryInfoList.add(createDictInfo(new int[] { 0, 1, 2 }));
+ dictionaryInfoList.add(createDictInfo(new int[] {}));
+ dictionaryInfoList.add(createDictInfo(new int[] { 7, 8, 9 }));
Integer[] values = enumerateDictInfoList(dictionaryInfoList);
assertEquals(6, values.length);
- assertArrayEquals(new Integer[]{0, 1, 2, 7, 8, 9}, values);
+ assertArrayEquals(new Integer[] { 0, 1, 2, 7, 8, 9 }, values);
}
@Test
public void testLastEmptyDicts() throws IOException {
List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(3);
- dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2}));
- dictionaryInfoList.add(createDictInfo(new int[]{6, 7, 8}));
- dictionaryInfoList.add(createDictInfo(new int[]{}));
+ dictionaryInfoList.add(createDictInfo(new int[] { 0, 1, 2 }));
+ dictionaryInfoList.add(createDictInfo(new int[] { 6, 7, 8 }));
+ dictionaryInfoList.add(createDictInfo(new int[] {}));
Integer[] values = enumerateDictInfoList(dictionaryInfoList);
assertEquals(6, values.length);
- assertArrayEquals(new Integer[]{0, 1, 2, 6, 7, 8}, values);
+ assertArrayEquals(new Integer[] { 0, 1, 2, 6, 7, 8 }, values);
}
@Test
public void testUnorderedDicts() throws IOException {
List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(3);
- dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 6}));
- dictionaryInfoList.add(createDictInfo(new int[]{3, 7, 8}));
- dictionaryInfoList.add(createDictInfo(new int[]{2, 7, 9}));
+ dictionaryInfoList.add(createDictInfo(new int[] { 0, 1, 6 }));
+ dictionaryInfoList.add(createDictInfo(new int[] { 3, 7, 8 }));
+ dictionaryInfoList.add(createDictInfo(new int[] { 2, 7, 9 }));
Integer[] values = enumerateDictInfoList(dictionaryInfoList);
assertEquals(9, values.length);
- assertArrayEquals(new Integer[]{0, 1, 2, 3, 6, 7, 7, 8, 9}, values);
+ assertArrayEquals(new Integer[] { 0, 1, 2, 3, 6, 7, 7, 8, 9 }, values);
}
-
-
-
public static class MockDictionary extends Dictionary<String> {
private static final long serialVersionUID = 1L;
-
+
public int[] values;
@Override
@@ -125,7 +122,7 @@ public class MultipleDictionaryValueEnumeratorTest {
@Override
public int getMaxId() {
- return values.length-1;
+ return values.length - 1;
}
@Override
@@ -148,16 +145,17 @@ public class MultipleDictionaryValueEnumeratorTest {
return "" + values[id];
}
-
@Override
- public void dump(PrintStream out) {}
+ public void dump(PrintStream out) {
+ }
@Override
- public void write(DataOutput out) throws IOException {}
+ public void write(DataOutput out) throws IOException {
+ }
@Override
- public void readFields(DataInput in) throws IOException {}
-
+ public void readFields(DataInput in) throws IOException {
+ }
@Override
public boolean contains(Dictionary another) {