You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/07/26 03:33:44 UTC
[25/55] [abbrv] kylin git commit: KYLIN-1894 fix test and cleanup tmp
files
KYLIN-1894 fix test and cleanup tmp files
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28a2b9f7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28a2b9f7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28a2b9f7
Branch: refs/heads/1.5.x-CDH5.7
Commit: 28a2b9f7eadc05fdf4e0a9e268b0adb142bdfccf
Parents: f0db336
Author: sunyerui <su...@gmail.com>
Authored: Mon Jul 18 17:50:12 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 18 18:37:46 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/AppendTrieDictionary.java | 2 +-
.../kylin/dict/AppendTrieDictionaryTest.java | 16 +++++++--
.../apache/kylin/dict/CachedTreeMapTest.java | 34 +++++++++++++-------
3 files changed, 36 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/28a2b9f7/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 4cce586..14980bf 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -950,8 +950,8 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
if (dict == null) {
dict = new AppendTrieDictionary<T>();
}
- dict.flushIndex((CachedTreeMap) mutableDictSliceMap);
dict.update(baseDir, baseId, maxId, maxValueLength, nValues, bytesConverter, (CachedTreeMap)mutableDictSliceMap);
+ dict.flushIndex((CachedTreeMap) mutableDictSliceMap);
return dict;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/28a2b9f7/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index b81a439..4266f2a 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -26,6 +26,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
+import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -49,6 +50,8 @@ import org.junit.Test;
*/
public class AppendTrieDictionaryTest {
+ public static final String BASE_DIR = "/tmp/kylin_append_dict";
+
@BeforeClass
public static void setUp() {
KylinConfig.destroyInstance();
@@ -56,7 +59,7 @@ public class AppendTrieDictionaryTest {
KylinConfig config = KylinConfig.getInstanceFromEnv();
config.setAppendDictEntrySize(50000);
config.setAppendDictCacheSize(3);
- config.setProperty("kylin.hdfs.working.dir", "/tmp/kylin_append_dict");
+ config.setProperty("kylin.hdfs.working.dir", BASE_DIR);
}
@AfterClass
@@ -66,6 +69,13 @@ public class AppendTrieDictionaryTest {
FileSystem.get(new Path(workingDir).toUri(), new Configuration()).delete(new Path(workingDir), true);
} catch (IOException e) {
}
+ File tmpLocalDir = new File(BASE_DIR);
+ if (tmpLocalDir.exists()) {
+ for (File f : tmpLocalDir.listFiles()) {
+ f.delete();
+ }
+ tmpLocalDir.delete();
+ }
}
public static final String[] words = new String[] { "paint", "par", "part", "parts", "partition", "partitions", "party", "partie", "parties", "patient", "taste", "tar", "trie", "try", "tries", "\u5b57\u5178", "\u5b57\u5178\u6811", "\u5b57\u6bcd", // non-ascii characters
@@ -122,7 +132,7 @@ public class AppendTrieDictionaryTest {
@Test
public void testHugeKeySet() throws IOException {
BytesConverter converter = new StringBytesConverter();
- AppendTrieDictionary.Builder<String> b = AppendTrieDictionary.Builder.create("/tmp/kylin_append_dict");
+ AppendTrieDictionary.Builder<String> b = AppendTrieDictionary.Builder.create(BASE_DIR);
AppendTrieDictionary<String> dict = null;
InputStream is = new FileInputStream("src/test/resources/dict/huge_key");
@@ -152,7 +162,7 @@ public class AppendTrieDictionaryTest {
}
BytesConverter converter = new StringBytesConverter();
- AppendTrieDictionary.Builder<String> b = AppendTrieDictionary.Builder.create("/tmp/kylin_append_dict");
+ AppendTrieDictionary.Builder<String> b = AppendTrieDictionary.Builder.create(BASE_DIR);
AppendTrieDictionary<String> dict = null;
TreeMap<Integer, String> checkMap = new TreeMap<>();
int firstAppend = rnd.nextInt(strList.size() / 2);
http://git-wip-us.apache.org/repos/asf/kylin/blob/28a2b9f7/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
index d2af621..3366bf6 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
@@ -3,6 +3,7 @@ package org.apache.kylin.dict;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
@@ -81,8 +82,7 @@ public class CachedTreeMapTest {
public static final String backupDir = "/tmp/kylin_cachedtreemap_test.bak/";
public static final String tmpDir = "/tmp/kylin_cachedtreemap_test.tmp/";
- @After
- public void afterTest() {
+ private static void cleanup() {
File dir = new File(baseDir);
if (dir.exists()) {
for (File f : dir.listFiles()) {
@@ -110,9 +110,19 @@ public class CachedTreeMapTest {
VALUE_WRITE_ERROR_TOGGLE = false;
}
+ @After
+ public void afterTest() {
+ cleanup();
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ cleanup();
+ }
+
@Test
public void testCachedTreeMap() throws IOException {
- CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir)
+ CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
.persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
map.put(Key.of(1), Value.of("a"));
map.put(Key.of(2), Value.of("b"));
@@ -133,7 +143,7 @@ public class CachedTreeMapTest {
assertEquals(5, dir.listFiles(new CachedFileFilter()).length);
DataInputStream in = new DataInputStream(new FileInputStream(baseDir+".index"));
- CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir)
+ CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
.persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
map2.readFields(in);
assertEquals(5, map2.size());
@@ -152,7 +162,7 @@ public class CachedTreeMapTest {
@Test
public void testWriteFailed() throws IOException {
// normal case
- CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir)
+ CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
.persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
map.put(Key.of(1), Value.of("a"));
map.put(Key.of(2), Value.of("b"));
@@ -167,14 +177,14 @@ public class CachedTreeMapTest {
map.commit(false);
DataInputStream in = new DataInputStream(new FileInputStream(baseDir+".index"));
- CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir)
+ CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
.persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
map2.readFields(in);
assertEquals(3, map2.size());
assertEquals("a", ((Value)map2.get(Key.of(1))).valueStr);
// suppose write value failed and didn't commit data
- map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir)
+ map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
.persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
VALUE_WRITE_ERROR_TOGGLE = true;
map.put(Key.of(1), Value.of("aa"));
@@ -191,7 +201,7 @@ public class CachedTreeMapTest {
// read map data should not be modified
in = new DataInputStream(new FileInputStream(baseDir+".index"));
- map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir)
+ map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
.persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
map2.readFields(in);
assertEquals(3, map2.size());
@@ -203,7 +213,7 @@ public class CachedTreeMapTest {
@Test
public void testCommit() throws IOException {
- CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir)
+ CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
.persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
map.put(Key.of(1), Value.of("a"));
map.put(Key.of(2), Value.of("b"));
@@ -220,7 +230,7 @@ public class CachedTreeMapTest {
assertFalse(new File(backupDir).exists());
DataInputStream in = new DataInputStream(new FileInputStream(baseDir+".index"));
- CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir)
+ CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
.persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
map2.readFields(in);
assertEquals(4, map2.size());
@@ -242,7 +252,7 @@ public class CachedTreeMapTest {
assertEquals(4, new File(baseDir).listFiles(new CachedFileFilter()).length);
in = new DataInputStream(new FileInputStream(baseDir+".index"));
- map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir)
+ map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
.persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
map2.readFields(in);
assertEquals(4, map2.size());
@@ -254,7 +264,7 @@ public class CachedTreeMapTest {
assertEquals(6, new File(baseDir).listFiles(new CachedFileFilter()).length);
in = new DataInputStream(new FileInputStream(baseDir+".index"));
- map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir)
+ map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
.persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
map2.readFields(in);
assertEquals(6, map2.size());