You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/02/04 00:51:40 UTC
[37/50] [abbrv] kylin git commit: minor, stablize
org.apache.kylin.dict.TrieDictionaryForestTest#englishWordsTest
minor, stablize org.apache.kylin.dict.TrieDictionaryForestTest#englishWordsTest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5eedd135
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5eedd135
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5eedd135
Branch: refs/heads/sync
Commit: 5eedd135d27da96c2242079fd9e188fce0518f90
Parents: 9198cfe
Author: lidongsjtu <li...@apache.org>
Authored: Sun Jan 28 19:55:07 2018 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sun Jan 28 19:55:07 2018 +0800
----------------------------------------------------------------------
.../kylin/dict/TrieDictionaryForestTest.java | 28 ++++++++++++++------
1 file changed, 20 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eedd135/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index 099075c..1f63111 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -43,6 +43,7 @@ import java.util.NoSuchElementException;
import java.util.Random;
import java.util.TreeSet;
+import org.apache.kylin.common.util.Bytes;
import org.junit.Ignore;
import org.junit.Test;
@@ -187,8 +188,14 @@ public class TrieDictionaryForestTest {
str.add("party");
str.add("parties");
str.add("paint");
- String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
+ String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
System.out.println("The length of the long string is " + longStr.length());
str.add(longStr);
@@ -740,7 +747,8 @@ public class TrieDictionaryForestTest {
System.out.println("times:" + i);
}
- System.out.println("compare build time. Old trie : " + oldDictTotalBuildTime / 1000.0 + "s.New trie : " + newDictTotalBuildTime / 1000.0 + "s");
+ System.out.println("compare build time. Old trie : " + oldDictTotalBuildTime / 1000.0 + "s.New trie : "
+ + newDictTotalBuildTime / 1000.0 + "s");
}
private void evaluateDataSize(ArrayList<String> list) {
@@ -806,7 +814,8 @@ public class TrieDictionaryForestTest {
benchmark("Benchmark", dict, set, map, strArray, array);
}
- private static int benchmark(String msg, TrieDictionaryForest<String> dict, TreeSet<String> set, HashMap<String, Integer> map, String[] strArray, byte[][] array) {
+ private static int benchmark(String msg, TrieDictionaryForest<String> dict, TreeSet<String> set,
+ HashMap<String, Integer> map, String[] strArray, byte[][] array) {
int n = set.size();
int times = Math.max(10 * 1000 * 1000 / n, 1); // run 10 million lookups
int keep = 0; // make sure JIT don't OPT OUT function calls under test
@@ -889,7 +898,7 @@ public class TrieDictionaryForestTest {
assertEquals(id, dict.getIdFromValue(value));
assertEquals(value, dict.getValueFromId(id));
- assertArrayEquals(value.getBytes(), dict.getValueByteFromId(id));
+ assertArrayEquals(Bytes.toBytes(value), dict.getValueByteFromId(id));
}
//test not found value
@@ -932,14 +941,16 @@ public class TrieDictionaryForestTest {
}
public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterable<String> strs, int baseId) {
- TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId);
+ TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(),
+ baseId);
for (String s : strs)
b.addValue(s);
return b;
}
public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterable<String> strs, int baseId, int treeSize) {
- TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId);
+ TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(),
+ baseId);
b.setMaxTrieTreeSize(treeSize);
for (String s : strs) {
b.addValue(s);
@@ -948,7 +959,8 @@ public class TrieDictionaryForestTest {
}
public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterator<String> strs, int baseId, int treeSize) {
- TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId);
+ TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(),
+ baseId);
b.setMaxTrieTreeSize(treeSize);
while (strs.hasNext())
b.addValue(strs.next());