You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/02/11 13:50:11 UTC
[32/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java b/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
deleted file mode 100644
index 0af7dfc..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.io.File;
-
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.dict.lookup.FileTable;
-import org.apache.kylin.dict.lookup.LookupBytesTable;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- */
-public class LookupTableTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void testBasic() throws Exception {
- TableDesc siteTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("EDW.TEST_SITES");
- TableDesc categoryTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.test_category_groupings");
- LookupBytesTable lookup;
-
- System.out.println("============================================================================");
-
- File f = new File(LOCALMETA_TEST_DATA + "/data/EDW.TEST_SITES.csv");
- lookup = new LookupBytesTable(siteTable, new String[] { "SITE_ID" }, new FileTable("file://" + f.getAbsolutePath(), 10));
- lookup.dump();
-
- System.out.println("============================================================================");
-
- f = new File(LOCALMETA_TEST_DATA + "/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv");
- lookup = new LookupBytesTable(categoryTable, new String[] { "leaf_categ_id", "site_id" }, new FileTable("file://" + f.getAbsolutePath(), 36));
- lookup.dump();
-
- System.out.println("============================================================================");
-
- ByteArray k1 = new ByteArray(Bytes.toBytes("533"));
- ByteArray k2 = new ByteArray(Bytes.toBytes("0"));
- Array<ByteArray> key = new Array<ByteArray>(new ByteArray[] { k1, k2 });
- System.out.println(lookup.getRow(key));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
deleted file mode 100644
index 4647ef8..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.math.BigDecimal;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.kylin.common.util.Bytes;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-/**
- * @author yangli9
- *
- */
-public class NumberDictionaryTest {
-
- NumberDictionary.NumberBytesCodec codec = new NumberDictionary.NumberBytesCodec();
- Random rand = new Random();
-
- @Test
- public void testNumberEncode() {
- checkCodec("12345", "00000000000012345");
- checkCodec("12345.123", "00000000000012345.123");
- checkCodec("-12345", "-9999999999987654;");
- checkCodec("-12345.123", "-9999999999987654.876;");
- checkCodec("0", "00000000000000000");
- checkCodec("0.0", "00000000000000000.0");
- }
-
- private void checkCodec(String number, String code) {
- assertEquals(code, encodeNumber(number));
- assertEquals(number, decodeNumber(code));
- }
-
- private String decodeNumber(String code) {
- byte[] buf = Bytes.toBytes(code);
- System.arraycopy(buf, 0, codec.buf, 0, buf.length);
- codec.bufOffset = 0;
- codec.bufLen = buf.length;
- int len = codec.decodeNumber(buf, 0);
- return Bytes.toString(buf, 0, len);
- }
-
- private String encodeNumber(String number) {
- byte[] num1 = Bytes.toBytes(number);
- codec.encodeNumber(num1, 0, num1.length);
- return Bytes.toString(codec.buf, codec.bufOffset, codec.bufLen);
- }
-
- @Test
- public void testDictionary() {
- int n = 100;
-
- Set<BigDecimal> set = Sets.newHashSet();
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new StringBytesConverter());
- for (int i = 0; i < n; i++) {
- String num = randNumber();
- if (set.add(new BigDecimal(num))) {
- builder.addValue(num);
- }
- }
-
- List<BigDecimal> sorted = Lists.newArrayList();
- sorted.addAll(set);
- Collections.sort(sorted);
-
- // test exact match
- NumberDictionary<String> dict = builder.build(0);
- for (int i = 0; i < sorted.size(); i++) {
- String dictNum = dict.getValueFromId(i);
- System.out.println(sorted.get(i) + "\t" + dictNum);
- }
-
- for (int i = 0; i < sorted.size(); i++) {
- String dictNum = dict.getValueFromId(i);
- assertEquals(sorted.get(i), new BigDecimal(dictNum));
- }
-
- // test rounding
- for (int i = 0; i < n; i++) {
- String randStr = randNumber();
- BigDecimal rand = new BigDecimal(randStr);
- int binarySearch = Collections.binarySearch(sorted, rand);
- if (binarySearch >= 0)
- continue;
- int insertion = -(binarySearch + 1);
- int expectedLowerId = insertion - 1;
- int expectedHigherId = insertion;
- // System.out.println("-- " + randStr + ", " + expectedLowerId +
- // ", " + expectedHigherId);
-
- if (expectedLowerId < 0) {
- try {
- dict.getIdFromValue(randStr, -1);
- fail();
- } catch (IllegalArgumentException ex) {
- // expect
- }
- } else {
- assertEquals(expectedLowerId, dict.getIdFromValue(randStr, -1));
- }
-
- if (expectedHigherId >= sorted.size()) {
- try {
- dict.getIdFromValue(randStr, 1);
- fail();
- } catch (IllegalArgumentException ex) {
- // expect
- }
- } else {
- assertEquals(expectedHigherId, dict.getIdFromValue(randStr, 1));
- }
- }
- }
-
- private String randNumber() {
- int digits1 = rand.nextInt(10);
- int digits2 = rand.nextInt(3);
- int sign = rand.nextInt(2);
- if (digits1 == 0 && digits2 == 0) {
- return randNumber();
- }
- StringBuilder buf = new StringBuilder();
- if (sign == 1)
- buf.append("-");
- for (int i = 0; i < digits1; i++)
- buf.append("" + rand.nextInt(10));
- if (digits2 > 0) {
- buf.append(".");
- for (int i = 0; i < digits2; i++)
- buf.append("" + rand.nextInt(9) + 1); // BigDecimal thinks 4.5
- // != 4.50, my god!
- }
- return buf.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java b/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
deleted file mode 100644
index c8dddde..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.dict.lookup.HiveTable;
-import org.apache.kylin.dict.lookup.ReadableTable.TableReader;
-import org.apache.kylin.dict.lookup.SnapshotManager;
-import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-public class SnapshotManagerTest extends HBaseMetadataTestCase {
-
- SnapshotManager snapshotMgr;
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- snapshotMgr = SnapshotManager.getInstance(getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void basicTest() throws Exception {
- String tableName = "EDW.TEST_SITES";
- HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName);
- TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName);
- String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath();
-
- snapshotMgr.wipeoutCache();
-
- SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath);
-
- // compare hive & snapshot
- TableReader hiveReader = hiveTable.getReader();
- TableReader snapshotReader = snapshot.getReader();
-
- while (true) {
- boolean hiveNext = hiveReader.next();
- boolean snapshotNext = snapshotReader.next();
- assertEquals(hiveNext, snapshotNext);
-
- if (hiveNext == false)
- break;
-
- String[] hiveRow = hiveReader.getRow();
- String[] snapshotRow = snapshotReader.getRow();
- assertArrayEquals(hiveRow, snapshotRow);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java b/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
deleted file mode 100644
index 0d1999a..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.kylin.dict.lookup.FileTable;
-import org.apache.kylin.dict.lookup.FileTableReader;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-public class TableReaderTest {
-
- @Test
- public void testBasicReader() throws IOException {
- File f = new File("src/test/resources/dict/DW_SITES");
- FileTableReader reader = new FileTableReader("file://" + f.getAbsolutePath(), FileTable.DELIM_AUTO, 10);
- while (reader.next()) {
- assertEquals("[-1, Korea Auction.co.kr, S, 48, 0, 111, 2009-02-11, , DW_OFFPLAT, ]", Arrays.toString(reader.getRow()));
- break;
- }
- reader.close();
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
deleted file mode 100644
index cb15731..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ /dev/null
@@ -1,343 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Random;
-import java.util.TreeSet;
-
-import org.junit.Test;
-
-public class TrieDictionaryTest {
-
- public static void main(String[] args) throws Exception {
- InputStream is = new FileInputStream("src/test/resources/dict/dw_category_grouping_names.dat");
- // InputStream is =
- // Util.getPackageResourceAsStream(TrieDictionaryTest.class,
- // "eng_com.dic");
- ArrayList<String> str = loadStrings(is);
- benchmarkStringDictionary(str);
- }
-
- @Test
- public void partOverflowTest() {
- ArrayList<String> str = new ArrayList<String>();
- // str.add("");
- str.add("part");
- str.add("par");
- str.add("partition");
- str.add("party");
- str.add("parties");
- str.add("paint");
- String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
- System.out.println("The length of the long string is " + longStr.length());
- str.add(longStr);
-
- str.add("zzzzzz" + longStr);// another long string
-
- TrieDictionaryBuilder<String> b = newDictBuilder(str);
- TrieDictionary<String> dict = b.build(0);
-
- TreeSet<String> set = new TreeSet<String>();
- for (String s : str) {
- set.add(s);
- }
-
- // test serialize
- dict = testSerialize(dict);
-
- // test basic id<==>value
- Iterator<String> it = set.iterator();
- int id = 0;
- int previousId = -1;
- for (; it.hasNext(); id++) {
- String value = it.next();
-
- // in case of overflow parts, there exist interpolation nodes
- // they exist to make sure that any node's part is shorter than 255
- int actualId = dict.getIdFromValue(value);
- assertTrue(actualId >= id);
- assertTrue(actualId > previousId);
- previousId = actualId;
-
- assertEquals(value, dict.getValueFromId(actualId));
- }
- }
-
- @Test
- public void emptyValueTest() {
- ArrayList<String> str = new ArrayList<String>();
- str.add("");
- str.add("part");
- str.add("par");
- str.add("partition");
- str.add("party");
- str.add("parties");
- str.add("paint");
- testStringDictionary(str, null);
- }
-
- @Test
- public void simpleTrieTest() {
- ArrayList<String> str = new ArrayList<String>();
- str.add("part");
- str.add("part"); // meant to be dup
- str.add("par");
- str.add("partition");
- str.add("party");
- str.add("parties");
- str.add("paint");
-
- ArrayList<String> notFound = new ArrayList<String>();
- notFound.add("");
- notFound.add("p");
- notFound.add("pa");
- notFound.add("pb");
- notFound.add("parti");
- notFound.add("partz");
- notFound.add("partyz");
-
- testStringDictionary(str, notFound);
- }
-
- @Test
- public void englishWordsTest() throws Exception {
- InputStream is = new FileInputStream("src/test/resources/dict/english-words.80 (scowl-2015.05.18).dic");
- ArrayList<String> str = loadStrings(is);
- testStringDictionary(str, null);
- }
-
- @Test
- public void categoryNamesTest() throws Exception {
- InputStream is = new FileInputStream("src/test/resources/dict/dw_category_grouping_names.dat");
- ArrayList<String> str = loadStrings(is);
- testStringDictionary(str, null);
- }
-
- private static void benchmarkStringDictionary(ArrayList<String> str) throws UnsupportedEncodingException {
- TrieDictionaryBuilder<String> b = newDictBuilder(str);
- b.stats().print();
- TrieDictionary<String> dict = b.build(0);
-
- TreeSet<String> set = new TreeSet<String>();
- for (String s : str) {
- set.add(s);
- }
-
- // prepare id==>value array and value==>id map
- HashMap<String, Integer> map = new HashMap<String, Integer>();
- String[] strArray = new String[set.size()];
- byte[][] array = new byte[set.size()][];
- Iterator<String> it = set.iterator();
- for (int id = 0; it.hasNext(); id++) {
- String value = it.next();
- map.put(value, id);
- strArray[id] = value;
- array[id] = value.getBytes("UTF-8");
- }
-
- // System.out.println("Dict size in bytes: " +
- // MemoryUtil.deepMemoryUsageOf(dict));
- // System.out.println("Map size in bytes: " +
- // MemoryUtil.deepMemoryUsageOf(map));
- // System.out.println("Array size in bytes: " +
- // MemoryUtil.deepMemoryUsageOf(strArray));
-
- // warm-up, said that code only got JIT after run 1k-10k times,
- // following jvm options may help
- // -XX:CompileThreshold=1500
- // -XX:+PrintCompilation
- benchmark("Warm up", dict, set, map, strArray, array);
- benchmark("Benchmark", dict, set, map, strArray, array);
- }
-
- private static int benchmark(String msg, TrieDictionary<String> dict, TreeSet<String> set, HashMap<String, Integer> map, String[] strArray, byte[][] array) {
- int n = set.size();
- int times = 10 * 1000 * 1000 / n; // run 10 million lookups
- int keep = 0; // make sure JIT don't OPT OUT function calls under test
- byte[] valueBytes = new byte[dict.getSizeOfValue()];
- long start;
-
- // benchmark value==>id, via HashMap
- System.out.println(msg + " HashMap lookup value==>id");
- start = System.currentTimeMillis();
- for (int i = 0; i < times; i++) {
- for (int j = 0; j < n; j++) {
- keep |= map.get(strArray[j]);
- }
- }
- long timeValueToIdByMap = System.currentTimeMillis() - start;
- System.out.println(timeValueToIdByMap);
-
- // benchmark value==>id, via Dict
- System.out.println(msg + " Dictionary lookup value==>id");
- start = System.currentTimeMillis();
- for (int i = 0; i < times; i++) {
- for (int j = 0; j < n; j++) {
- keep |= dict.getIdFromValueBytes(array[j], 0, array[j].length);
- }
- }
- long timeValueToIdByDict = System.currentTimeMillis() - start;
- System.out.println(timeValueToIdByDict);
-
- // benchmark id==>value, via Array
- System.out.println(msg + " Array lookup id==>value");
- start = System.currentTimeMillis();
- for (int i = 0; i < times; i++) {
- for (int j = 0; j < n; j++) {
- keep |= strArray[j].length();
- }
- }
- long timeIdToValueByArray = System.currentTimeMillis() - start;
- System.out.println(timeIdToValueByArray);
-
- // benchmark id==>value, via Dict
- System.out.println(msg + " Dictionary lookup id==>value");
- start = System.currentTimeMillis();
- for (int i = 0; i < times; i++) {
- for (int j = 0; j < n; j++) {
- keep |= dict.getValueBytesFromId(j, valueBytes, 0);
- }
- }
- long timeIdToValueByDict = System.currentTimeMillis() - start;
- System.out.println(timeIdToValueByDict);
-
- return keep;
- }
-
- private static void testStringDictionary(ArrayList<String> str, ArrayList<String> notFound) {
- TrieDictionaryBuilder<String> b = newDictBuilder(str);
- int baseId = new Random().nextInt(100);
- TrieDictionary<String> dict = b.build(baseId);
-
- TreeSet<String> set = new TreeSet<String>();
- for (String s : str) {
- set.add(s);
- }
-
- // test serialize
- dict = testSerialize(dict);
-
- // test basic id<==>value
- Iterator<String> it = set.iterator();
- int id = baseId;
- for (; it.hasNext(); id++) {
- String value = it.next();
- // System.out.println("checking " + id + " <==> " + value);
-
- assertEquals(id, dict.getIdFromValue(value));
- assertEquals(value, dict.getValueFromId(id));
- }
- if (notFound != null) {
- for (String s : notFound) {
- try {
- dict.getIdFromValue(s);
- fail("For not found value '" + s + "', IllegalArgumentException is expected");
- } catch (IllegalArgumentException e) {
- // good
- }
- }
- }
-
- // test null value
- int nullId = dict.getIdFromValue(null);
- assertNull(dict.getValueFromId(nullId));
- int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
- assertEquals(nullId, nullId2);
- }
-
- private static TrieDictionary<String> testSerialize(TrieDictionary<String> dict) {
- try {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- DataOutputStream dataout = new DataOutputStream(bout);
- dict.write(dataout);
- dataout.close();
- ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
- DataInputStream datain = new DataInputStream(bin);
- TrieDictionary<String> r = new TrieDictionary<String>();
- r.readFields(datain);
- datain.close();
- return r;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private static TrieDictionaryBuilder<String> newDictBuilder(ArrayList<String> str) {
- TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
- for (String s : str)
- b.addValue(s);
- return b;
- }
-
- private static ArrayList<String> loadStrings(InputStream is) throws Exception {
- ArrayList<String> r = new ArrayList<String>();
- BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
- try {
- String word;
- while ((word = reader.readLine()) != null) {
- word = word.trim();
- if (word.isEmpty() == false)
- r.add(word);
- }
- } finally {
- reader.close();
- is.close();
- }
- return r;
- }
-
- @Test
- public void testSuperLongStringValue() {
- String longPrefix = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789";
-
- TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
- String v1 = longPrefix + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz";
- String v2 = longPrefix + "xyz";
-
- b.addValue(v1);
- b.addValue(v2);
- TrieDictionary<String> dict = b.build(0);
- dict.dump(System.out);
- }
-
- @Test
- public void testRounding() {
- // see NumberDictionaryTest.testRounding();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
deleted file mode 100644
index 3d7267d..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Created by shaoshi on 2/3/15.
- */
-public class LookupTableTest extends LocalFileMetadataTestCase {
-
- private KylinConfig config = null;
-
- private LookupTable lookupTable;
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- config = KylinConfig.getInstanceFromEnv();
- lookupTable = initLookupTable();
- }
-
- @After
- public void tearDown() {
- cleanupTestMetadata();
- }
-
- @Test
- public void testScan() throws Exception {
- List<String> values = new ArrayList<String>();
- values.add("2012-01-24");
- values.add("2012-12-30");
- List<String> results = lookupTable.scan("CAL_DT", values, "YEAR_BEG_DT");
-
- Assert.assertTrue(results.size() > 0);
- for (String i : results) {
- System.out.println(i);
-
- Assert.assertEquals("2012-01-01", i);
- }
- }
-
- @Test
- public void testMapRange() throws Exception {
- List<String> values = new ArrayList<String>();
- values.add("2012-01-24");
- values.add("2012-12-30");
- Pair<String, String> results = lookupTable.mapRange("CAL_DT", "2012-01-24", "2012-12-30", "QTR_BEG_DT");
-
- Assert.assertTrue(results != null);
- System.out.println("The first qtr_beg_dt is " + results.getFirst());
- System.out.println("The last qtr_beg_dt is " + results.getSecond());
-
- Assert.assertEquals("2012-01-01", results.getFirst());
- Assert.assertEquals("2012-10-01", results.getSecond());
- }
-
- @Test
- public void testMapValues() throws Exception {
- Set<String> values = new HashSet<String>();
- values.add("2012-01-24");
- values.add("2012-12-30");
- Set<String> results = lookupTable.mapValues("CAL_DT", values, "YEAR_BEG_DT");
-
- Assert.assertTrue(results.size() == 1);
- for (String i : results) {
- System.out.println(i);
-
- Assert.assertEquals("2012-01-01", i);
- }
- }
-
- public LookupTable initLookupTable() throws Exception {
-
- MetadataManager metaMgr = MetadataManager.getInstance(config);
-
- String tableName = "EDW.TEST_CAL_DT";
- String[] pkCols = new String[] { "CAL_DT" };
- String snapshotResPath = "/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot";
- SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
- TableDesc tableDesc = metaMgr.getTableDesc(tableName);
- LookupTable lt = new LookupStringTable(tableDesc, pkCols, snapshot);
-
- System.out.println(lt);
-
- return lt;
- }
-
- private SnapshotManager getSnapshotManager() {
- return SnapshotManager.getInstance(config);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/resources/dict/DW_SITES
----------------------------------------------------------------------
diff --git a/dictionary/src/test/resources/dict/DW_SITES b/dictionary/src/test/resources/dict/DW_SITES
deleted file mode 100644
index 6f6af46..0000000
Binary files a/dictionary/src/test/resources/dict/DW_SITES and /dev/null differ