You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2020/01/12 09:57:21 UTC
[kylin] branch master updated: minor, add some ut for dictionaries
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new e37936e minor, add some ut for dictionaries
e37936e is described below
commit e37936e6764ec284f5ba8ade868c6db74fb10310
Author: etherge <et...@163.com>
AuthorDate: Sat Jan 11 05:38:04 2020 -0500
minor, add some ut for dictionaries
---
core-dictionary/pom.xml | 6 ++
.../org/apache/kylin/dict/TimeStrDictionary.java | 5 +-
.../kylin/dict/AppendTrieDictionaryTest.java | 79 +++++++++++++++-
.../apache/kylin/dict/DateStrDictionaryTest.java | 102 ++++++++++++++++++++
.../apache/kylin/dict/ShrunkenDictionaryTest.java | 105 ++++++++++++++++-----
.../apache/kylin/dict/TimeStrDictionaryTest.java | 74 ++++++++++++++-
pom.xml | 1 +
7 files changed, 337 insertions(+), 35 deletions(-)
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 06be88c..f143e65 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -55,6 +55,12 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava-testlib</artifactId>
+ <version>${guava-testlib.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
index d075ce1..afd7b17 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
@@ -20,7 +20,6 @@ package org.apache.kylin.dict;
import java.io.DataInput;
import java.io.DataOutput;
-import java.io.IOException;
import java.io.PrintStream;
import org.apache.kylin.common.util.DateFormat;
@@ -112,11 +111,11 @@ public class TimeStrDictionary extends Dictionary<String> {
}
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(DataOutput out) {
}
@Override
- public void readFields(DataInput in) throws IOException {
+ public void readFields(DataInput in) {
}
}
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index 7907da8..79e0925 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -61,6 +61,7 @@ import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.Lists;
+import com.google.common.testing.EqualsTester;
public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
private static final String RESOURCE_DIR = "/dict/append_dict_test/" + RandomUtil.randomUUID();
@@ -68,7 +69,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
private static String LOCAL_BASE_DIR;
@Before
- public void beforeTest() {
+ public void beforeTest() throws IOException {
staticCreateTestMetadata();
KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "50000");
BASE_DIR = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR
@@ -596,7 +597,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
public void testTooManySliceEvictions() throws IOException {
KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.max-cache-size", "3");
AppendTrieDictionaryBuilder builder = createBuilder();
- for (int i = 0 ; i < 100000; i++) {
+ for (int i = 0; i < 100000; i++) {
builder.addValue(Integer.toString(i));
}
AppendTrieDictionary dict = builder.build(0);
@@ -606,9 +607,8 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
assertEquals(0, dict.getCacheStats().evictionCount());
assertEquals(1, dict.getCacheStats().loadCount());
-
List<String> keys = new ArrayList<>(100000);
- for (int i = 0 ; i < 100000; i++) {
+ for (int i = 0; i < 100000; i++) {
keys.add(Integer.toString(i));
}
Collections.sort(keys);
@@ -633,4 +633,75 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.max-cache-size", "-1");
}
+
+ @Test
+ public void testEqualsAndHashCode() throws IOException {
+ AppendTrieDictionaryBuilder builder = createBuilder();
+ AppendTrieDictionary dict1 = builder.build(0);
+ AppendTrieDictionary dict2 = builder.build(1);
+
+ int maxEntriesPerSlice = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize();
+ String tempBaseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict"
+ + "/dict/append_dict_test/" + RandomUtil.randomUUID() + "/";
+ AppendTrieDictionaryBuilder builder2 = new AppendTrieDictionaryBuilder(tempBaseDir, maxEntriesPerSlice, true);
+ AppendTrieDictionary dict3 = builder2.build(0);
+ AppendTrieDictionary dict4 = builder2.build(1);
+
+ new EqualsTester().addEqualityGroup(dict1, dict2).addEqualityGroup(dict3, dict4).testEquals();
+ }
+
+ @Test
+ public void testToString() {
+ AppendTrieDictionary dict = createDefaultTestDict();
+ assertEquals("AppendTrieDictionary(" + BASE_DIR + ")", dict.toString());
+ }
+
+ @Test
+ public void testContains() {
+ AppendTrieDictionary dict = createDefaultTestDict();
+ assertEquals(false, dict.contains(dict));
+ }
+
+ @Test
+ public void testGetMinId() {
+ AppendTrieDictionary dict = createDefaultTestDict();
+ assertEquals(0, dict.getMinId());
+ }
+
+ @Test
+ public void testGetMaxId() throws IOException {
+ AppendTrieDictionaryBuilder builder = createBuilder();
+ builder.addValue("a");
+ builder.addValue("b");
+ builder.addValue("c");
+ builder.addValue("d");
+ builder.addValue("e");
+ builder.addValue("f");
+ AppendTrieDictionary dict = builder.build(0);
+ assertEquals(6, dict.getMaxId());
+ }
+
+ @Test
+ public void testGetSizeOfId() {
+ AppendTrieDictionary dict = createDefaultTestDict();
+ assertEquals(4, dict.getSizeOfId());
+ }
+
+ @Test
+ public void testGetSizeOfValue() throws IOException {
+ AppendTrieDictionaryBuilder builder = createBuilder();
+ builder.addValue("a");
+ builder.addValue("aaaaaa");
+ AppendTrieDictionary dict = builder.build(0);
+ assertEquals(6, dict.getSizeOfValue());
+ }
+
+ private AppendTrieDictionary createDefaultTestDict() {
+ try {
+ return createBuilder().build(0);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to prepare dict for testing.");
+ }
+
+ }
}
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
index c6d2e05..633dbe0 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
@@ -19,20 +19,122 @@
package org.apache.kylin.dict;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.dimension.DateDimEnc;
import org.junit.Before;
import org.junit.Test;
+import com.google.common.testing.EqualsTester;
+
public class DateStrDictionaryTest {
DateStrDictionary dict;
+ DateStrDictionary dictWithBaseId;
+ int baseId = 1;
+ static String datePattern = "yyyy-MM-dd";
@Before
public void setup() {
dict = new DateStrDictionary();
+ dictWithBaseId = new DateStrDictionary(datePattern, baseId);
+ }
+
+ @Test
+ public void testGetMinId() {
+ assertEquals(0, dict.getMinId());
+ assertEquals(baseId, dictWithBaseId.getMinId());
+ }
+
+ @Test
+ public void testGetMaxId() {
+ assertEquals(DateDimEnc.ID_9999_12_31, dict.getMaxId());
+ assertEquals(baseId + DateDimEnc.ID_9999_12_31, dictWithBaseId.getMaxId());
+ }
+
+ @Test
+ public void testGetSizeOfValue() {
+ assertEquals(DateFormat.DEFAULT_DATE_PATTERN.length(), dict.getSizeOfValue());
+ assertEquals(datePattern.length(), dictWithBaseId.getSizeOfValue());
+ }
+
+ @Test
+ public void testWrite() throws IOException {
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ DataOutputStream dataout = new DataOutputStream(bout);
+ dict.write(dataout);
+ dataout.close();
+ ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
+ DataInputStream datain = new DataInputStream(bin);
+
+ String datePattern = datain.readUTF();
+ int baseId = datain.readInt();
+ datain.close();
+
+ assertEquals(DateFormat.DEFAULT_DATE_PATTERN, datePattern);
+ assertEquals(baseId, 0);
+ }
+
+ @Test
+ public void testReadFields() throws IOException {
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ DataOutputStream dataout = new DataOutputStream(bout);
+ dataout.writeUTF(DateFormat.DEFAULT_DATE_PATTERN);
+ dataout.writeInt(0);
+ dataout.close();
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
+ DataInputStream datain = new DataInputStream(bin);
+ dict.readFields(datain);
+ datain.close();
+
+ assertEquals(DateFormat.DEFAULT_DATE_PATTERN.length(), dict.getSizeOfValue());
+ assertEquals(0, dict.getMinId());
+ test();
+ }
+
+ @Test
+ public void testEqualsAndHashCode() {
+ new EqualsTester().addEqualityGroup(dict, new DateStrDictionary(DateFormat.DEFAULT_DATE_PATTERN, 0))
+ .addEqualityGroup(dictWithBaseId).testEquals();
+ }
+
+ @Test
+ public void testContains() {
+ assertTrue(dict.contains(new DateStrDictionary()));
+ assertFalse(dict.contains(dictWithBaseId));
+ }
+
+ @Test
+ public void testDump() throws IOException {
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bout, false, StandardCharsets.UTF_8.name());
+ dict.dump(ps);
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
+ BufferedReader reader = new BufferedReader(new InputStreamReader(bin, StandardCharsets.UTF_8));
+ String output = reader.readLine();
+
+ bout.close();
+ bin.close();
+ ps.close();
+ reader.close();
+
+ assertEquals("DateStrDictionary [pattern=yyyy-MM-dd, baseId=0]", output);
}
@Test
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/ShrunkenDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/ShrunkenDictionaryTest.java
index c995d1b..b2981d4 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/ShrunkenDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/ShrunkenDictionaryTest.java
@@ -18,12 +18,16 @@
package org.apache.kylin.dict;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.List;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
@@ -33,9 +37,14 @@ import org.junit.BeforeClass;
import org.junit.Test;
public class ShrunkenDictionaryTest {
+ private static List<String> testData;
+ private static Dictionary originDict;
+ private static Dictionary shrunkenDict;
+
@BeforeClass
public static void setUp() {
LocalFileMetadataTestCase.staticCreateTestMetadata();
+ prepareTestData();
}
@AfterClass
@@ -45,28 +54,6 @@ public class ShrunkenDictionaryTest {
@Test
public void testStringDictionary() {
- ArrayList<String> strList = new ArrayList<String>();
- strList.add("");
- strList.add("part");
- strList.add("par");
- strList.add("partition");
- strList.add("party");
- strList.add("parties");
- strList.add("paint");
-
- TrieDictionaryBuilder<String> dictBuilder = new TrieDictionaryBuilder<>(new StringBytesConverter());
- for (String str : strList) {
- dictBuilder.addValue(str);
- }
- Dictionary<String> dict = dictBuilder.build(0);
-
- ShrunkenDictionary.StringValueSerializer valueSerializer = new ShrunkenDictionary.StringValueSerializer();
- ShrunkenDictionaryBuilder<String> shrunkenDictBuilder = new ShrunkenDictionaryBuilder<>(dict);
- for (int i = 0; i < strList.size(); i += 2) {
- shrunkenDictBuilder.addValue(strList.get(i));
- }
- Dictionary<String> shrunkenDict = shrunkenDictBuilder.build(valueSerializer);
-
try {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(bos);
@@ -76,14 +63,82 @@ public class ShrunkenDictionaryTest {
ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
DataInputStream dis = new DataInputStream(bis);
+ ShrunkenDictionary.StringValueSerializer valueSerializer = new ShrunkenDictionary.StringValueSerializer();
Dictionary<String> dShrunkenDict = new ShrunkenDictionary<>(valueSerializer);
dShrunkenDict.readFields(dis);
- for (int i = 0; i < strList.size(); i += 2) {
- String value = strList.get(i);
- Assert.assertEquals(dict.getIdFromValue(value), dShrunkenDict.getIdFromValue(value));
+ for (int i = 0; i < testData.size(); i += 2) {
+ String value = testData.get(i);
+ Assert.assertEquals(originDict.getIdFromValue(value), dShrunkenDict.getIdFromValue(value));
}
} catch (IOException e) {
}
}
+
+ @Test
+ public void testGetMinId() {
+ assertEquals(0, shrunkenDict.getMinId());
+ }
+
+ @Test
+ public void testGetMaxId() {
+ assertEquals(6, shrunkenDict.getMaxId());
+ }
+
+ @Test
+ public void testGetSizeOfId() {
+ assertEquals(1, shrunkenDict.getSizeOfId());
+ }
+
+ @Test
+ public void testGetSizeOfValue() {
+ assertEquals(9, shrunkenDict.getSizeOfValue());
+ }
+
+ @Test
+ public void testContains() {
+ assertFalse(shrunkenDict.contains(originDict));
+ }
+
+ @Test
+ public void testGetValueFromIdImpl() {
+ for (int i = 0; i < testData.size(); i += 2) {
+ assertEquals(testData.get(i), shrunkenDict.getValueFromId(originDict.getIdFromValue(testData.get(i))));
+ }
+ }
+
+ private static void prepareTestData() {
+ testData = new ArrayList<>();
+ testData.add("");
+ testData.add("part");
+ testData.add("par");
+ testData.add("partition");
+ testData.add("party");
+ testData.add("parties");
+ testData.add("paint");
+
+ originDict = constructOriginDict();
+ ShrunkenDictionary.StringValueSerializer valueSerializer = new ShrunkenDictionary.StringValueSerializer();
+ shrunkenDict = constructShrunkenDict(originDict, valueSerializer);
+ }
+
+ private static Dictionary constructOriginDict() {
+ TrieDictionaryBuilder<String> dictBuilder = new TrieDictionaryBuilder<>(new StringBytesConverter());
+ for (String str : testData) {
+ dictBuilder.addValue(str);
+ }
+ Dictionary<String> dict = dictBuilder.build(0);
+ return dict;
+ }
+
+ private static Dictionary constructShrunkenDict(Dictionary dictionary,
+ ShrunkenDictionary.ValueSerializer valueSerializer) {
+ ShrunkenDictionaryBuilder<String> shrunkenDictBuilder = new ShrunkenDictionaryBuilder<>(dictionary);
+ for (int i = 0; i < testData.size(); i += 2) {
+ System.out.println(testData.get(i));
+ shrunkenDictBuilder.addValue(testData.get(i));
+ }
+ Dictionary<String> shrunkenDict = shrunkenDictBuilder.build(valueSerializer);
+ return shrunkenDict;
+ }
}
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
index e852d26..a945bd4 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
@@ -18,13 +18,25 @@
package org.apache.kylin.dict;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+
import org.apache.kylin.common.util.DateFormat;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import com.google.common.testing.EqualsTester;
+
/**
*/
public class TimeStrDictionaryTest {
@@ -42,9 +54,15 @@ public class TimeStrDictionaryTest {
int c = dict.getIdFromValue("1999-01-01 00:00:00.000");
int d = dict.getIdFromValue("1999-01-01 00:00:00.022");
- Assert.assertEquals(a, b);
- Assert.assertEquals(a, c);
- Assert.assertEquals(a, d);
+ assertEquals(a, b);
+ assertEquals(a, c);
+ assertEquals(a, d);
+ }
+
+ @Test
+ public void testGetValueFromIdImpl() {
+ String value = dict.getValueFromIdImpl(0xffffffff);
+ assertEquals(null, value);
}
@Test
@@ -80,4 +98,54 @@ public class TimeStrDictionaryTest {
Assert.assertEquals(originChoppingMilis, back);
}
+ @Test
+ public void testGetMinId() {
+ assertEquals(0, dict.getMinId());
+ }
+
+ @Test
+ public void testGetMaxId() {
+ assertEquals(Integer.MAX_VALUE - 1, dict.getMaxId());
+ }
+
+ @Test
+ public void testGetSizeOfId() {
+ assertEquals(4, dict.getSizeOfId());
+ }
+
+ @Test
+ public void testGetSizeOfValue() {
+ assertEquals(19, dict.getSizeOfValue());
+ }
+
+ @Test
+ public void testEqualsAndHashCode() {
+ new EqualsTester().addEqualityGroup(dict, new TimeStrDictionary()).addEqualityGroup(new String("invalid"))
+ .testEquals();
+ }
+
+ @Test
+ public void testContains() {
+ assertTrue(dict.contains(new TimeStrDictionary()));
+ }
+
+ @Test
+ public void testDump() throws IOException {
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bout, false, StandardCharsets.UTF_8.name());
+ dict.dump(ps);
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
+ BufferedReader reader = new BufferedReader(new InputStreamReader(bin, StandardCharsets.UTF_8));
+ String output = reader.readLine();
+
+ bout.close();
+ bin.close();
+ ps.close();
+ reader.close();
+
+ assertEquals(
+ "TimeStrDictionary supporting from 1970-01-01 00:00:00 to 2038/01/19 03:14:07 (does not support millisecond)",
+ output);
+ }
}
diff --git a/pom.xml b/pom.xml
index fe97fab..3a7ed26 100644
--- a/pom.xml
+++ b/pom.xml
@@ -123,6 +123,7 @@
<mockito.version>2.7.14</mockito.version>
<mockito-all.version>1.9.5</mockito-all.version>
<powermock.version>1.7.0</powermock.version>
+ <guava-testlib.version>28.2-jre</guava-testlib.version>
<!-- Commons -->
<commons-lang3.version>3.4</commons-lang3.version>