You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by jh...@apache.org on 2015/12/10 07:48:23 UTC
tajo git commit: TAJO-2016: Investigate storage tests hangs.
Repository: tajo
Updated Branches:
refs/heads/master 3d59eebd5 -> 394879e0f
TAJO-2016: Investigate storage tests hangs.
Closes #906
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/394879e0
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/394879e0
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/394879e0
Branch: refs/heads/master
Commit: 394879e0f01273b2c8a66aef01d6cc313a5e5f3d
Parents: 3d59eeb
Author: Jinho Kim <jh...@apache.org>
Authored: Thu Dec 10 15:46:33 2015 +0900
Committer: Jinho Kim <jh...@apache.org>
Committed: Thu Dec 10 15:46:33 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
.../tajo/storage/TestByteBufLineReader.java | 7 +++---
.../tajo/storage/TestCompressionStorages.java | 15 ++++++------
.../apache/tajo/storage/TestFileTablespace.java | 25 ++++++++++----------
.../tajo/storage/raw/TestDirectRawFile.java | 2 +-
5 files changed, 27 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/394879e0/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index c859b18..25a4791 100644
--- a/CHANGES
+++ b/CHANGES
@@ -57,6 +57,8 @@ Release 0.12.0 - unreleased
BUG FIXES
+ TAJO-2016: Fix storage tests hangs. (jinho)
+
TAJO-2013: FilterPushDownRule fails due to the unsupported index. (jinho)
TAJO-2014: TestRpcClientManager fails occasionally. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/394879e0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
index b6f65df..8472095 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java
@@ -85,7 +85,7 @@ public class TestByteBufLineReader {
fs.close();
}
- @Test
+ @Test(timeout = 60000)
public void testReaderWithDFS() throws Exception {
final Configuration conf = new HdfsConfiguration();
String testDataPath = TEST_PATH + "/" + UUID.randomUUID().toString();
@@ -94,8 +94,7 @@ public class TestByteBufLineReader {
conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, true);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(2).build();
- cluster.waitClusterUp();
+ .numDataNodes(2).waitSafeMode(true).build();
TajoConf tajoConf = new TajoConf(conf);
tajoConf.setVar(TajoConf.ConfVars.ROOT_DIR, cluster.getFileSystem().getUri() + "/tajo");
@@ -122,7 +121,7 @@ public class TestByteBufLineReader {
lineReader.close();
fs.close();
} finally {
- cluster.shutdown(true);
+ cluster.shutdown();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/394879e0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
index 7836730..b63b497 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.compress.*;
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
import org.apache.hadoop.util.NativeCodeLoader;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -68,9 +69,9 @@ public class TestCompressionStorages {
@Parameterized.Parameters
public static Collection<Object[]> generateParameters() {
return Arrays.asList(new Object[][]{
- {"TEXT"},
- {"RCFILE"},
- {"SEQUENCEFILE"}
+ {BuiltinStorages.TEXT},
+ {BuiltinStorages.RCFILE},
+ {BuiltinStorages.SEQUENCE_FILE}
});
}
@@ -81,11 +82,11 @@ public class TestCompressionStorages {
@Test
public void testGzipCodecCompressionData() throws IOException {
- if (dataFormat.equalsIgnoreCase("RCFILE")) {
+ if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) {
if( ZlibFactory.isNativeZlibLoaded(conf)) {
storageCompressionTest(dataFormat, GzipCodec.class);
}
- } else if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) {
+ } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
if( ZlibFactory.isNativeZlibLoaded(conf)) {
storageCompressionTest(dataFormat, GzipCodec.class);
}
@@ -131,7 +132,7 @@ public class TestCompressionStorages {
extension = ((DelimitedTextFile.DelimitedTextFileAppender) appender).getExtension();
}
- int tupleNum = 100000;
+ int tupleNum = 1000;
VTuple vTuple;
for (int i = 0; i < tupleNum; i++) {
@@ -154,7 +155,7 @@ public class TestCompressionStorages {
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, tablets[0], schema);
scanner.init();
- if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) {
+ if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
assertTrue(scanner instanceof SequenceFileScanner);
Writable key = ((SequenceFileScanner) scanner).getKey();
assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());
http://git-wip-us.apache.org/repos/asf/tajo/blob/394879e0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
index d599a25..c99e0dc 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.*;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -41,7 +42,6 @@ import org.junit.Test;
import java.io.IOException;
import java.net.URI;
import java.util.List;
-import java.util.Optional;
import java.util.UUID;
import static org.junit.Assert.*;
@@ -70,7 +70,7 @@ public class TestFileTablespace {
schema.addColumn("age",Type.INT4);
schema.addColumn("name",Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
VTuple[] tuples = new VTuple[4];
for(int i=0; i < tuples.length; i++) {
@@ -82,7 +82,7 @@ public class TestFileTablespace {
Path path = StorageUtil.concatPath(testDir, "testGetScannerAndAppender", "table.csv");
localFs.mkdirs(path.getParent());
- FileTablespace fileStorageManager = (FileTablespace) TablespaceManager.getLocalFs();
+ FileTablespace fileStorageManager = TablespaceManager.getLocalFs();
assertEquals(localFs.getUri(), fileStorageManager.getFileSystem().getUri());
Appender appender = fileStorageManager.getAppender(meta, schema, path);
@@ -99,6 +99,7 @@ public class TestFileTablespace {
i++;
}
assertEquals(4,i);
+ localFs.delete(path, true);
}
@Test(timeout = 60000)
@@ -137,7 +138,7 @@ public class TestFileTablespace {
schema.addColumn("id", Type.INT4);
schema.addColumn("age",Type.INT4);
schema.addColumn("name",Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
List<Fragment> splits = Lists.newArrayList();
// Get FileFragments in partition batch
@@ -154,7 +155,7 @@ public class TestFileTablespace {
assertEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]);
fs.close();
} finally {
- cluster.shutdown(true);
+ cluster.shutdown();
}
}
@@ -203,7 +204,7 @@ public class TestFileTablespace {
assertEquals(0, splits.size());
fs.close();
} finally {
- cluster.shutdown(true);
+ cluster.shutdown();
}
}
@@ -254,7 +255,7 @@ public class TestFileTablespace {
assertNotEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]);
fs.close();
} finally {
- cluster.shutdown(true);
+ cluster.shutdown();
}
}
@@ -271,7 +272,6 @@ public class TestFileTablespace {
new MiniDFSCluster.Builder(hdfsConf).numDataNodes(1).build();
URI uri = URI.create(cluster.getFileSystem().getUri() + "/tajo");
- Optional<Tablespace> existingTs = Optional.empty();
try {
/* Local FileSystem */
FileTablespace space = TablespaceManager.getLocalFs();
@@ -279,16 +279,17 @@ public class TestFileTablespace {
FileTablespace distTablespace = new FileTablespace("testGetFileTablespace", uri, null);
distTablespace.init(conf);
- existingTs = TablespaceManager.addTableSpaceForTest(distTablespace);
+
+ TablespaceManager.addTableSpaceForTest(distTablespace);
/* Distributed FileSystem */
- space = (FileTablespace) TablespaceManager.get(uri);
+ space = TablespaceManager.get(uri);
assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri());
- space = (FileTablespace) TablespaceManager.getByName("testGetFileTablespace");
+ space = TablespaceManager.getByName("testGetFileTablespace");
assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri());
} finally {
- cluster.shutdown(true);
+ cluster.shutdown();
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/394879e0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
index 179c9eb..78e3390 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
@@ -117,7 +117,7 @@ public class TestDirectRawFile {
@AfterClass
public static void tearDownClass() throws InterruptedException {
- cluster.shutdown(true);
+ cluster.shutdown();
}
public Path getTestDir(FileSystem fs, String dir) throws IOException {