You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2014/07/15 09:16:47 UTC
[08/19] git commit: TAJO-852: Integration test using HCatalog as a
catalog store is failed. (jinho)
TAJO-852: Integration test using HCatalog as a catalog store is failed. (jinho)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/a17bd0f1
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/a17bd0f1
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/a17bd0f1
Branch: refs/heads/index_support
Commit: a17bd0f182d5405450b3e47be81f45dd6cdc362e
Parents: 95292d2
Author: jinossy <ji...@gmail.com>
Authored: Fri Jul 11 17:05:38 2014 +0900
Committer: jinossy <ji...@gmail.com>
Committed: Fri Jul 11 17:05:38 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../tajo/catalog/store/HCatalogStore.java | 25 ++-
.../catalog/store/HCatalogStoreClientPool.java | 32 ++-
.../java/org/apache/tajo/util/KeyValueSet.java | 2 +-
.../tajo/engine/query/TestCreateTable.java | 199 +++++++++++++------
.../org/apache/tajo/jdbc/TestResultSet.java | 3 +
.../java/org/apache/tajo/rpc/TestAsyncRpc.java | 22 +-
7 files changed, 182 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 43075f0..15bf988 100644
--- a/CHANGES
+++ b/CHANGES
@@ -82,6 +82,9 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-852: Integration test using HCatalog as a catalog store is failed.
+ (jinho)
+
TAJO-934: Multiple DISTINCT returns null grouping key value.
(Hyoungjun Kim via hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 3008ed9..6f48348 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -82,17 +82,15 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
// get table
try {
- try {
- client = clientPool.getClient();
- table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName);
- if (table != null) {
- exist = true;
- }
- } catch (NoSuchObjectException nsoe) {
- exist = false;
- } catch (Exception e) {
- throw new CatalogException(e);
+ client = clientPool.getClient();
+ table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName);
+ if (table != null) {
+ exist = true;
}
+ } catch (NoSuchObjectException nsoe) {
+ exist = false;
+ } catch (Exception e) {
+ throw new CatalogException(e);
} finally {
if (client != null) {
client.release();
@@ -168,6 +166,8 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
stats = new TableStats();
options = new KeyValueSet();
options.putAll(table.getParameters());
+ options.delete("EXTERNAL");
+
Properties properties = table.getMetadata();
if (properties != null) {
// set field delimiter
@@ -186,6 +186,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
} else {
nullFormat = "\\N";
}
+ options.delete(serdeConstants.SERIALIZATION_NULL_FORMAT);
// set file output format
String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT);
@@ -260,8 +261,10 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
if(client != null) client.release();
}
TableMeta meta = new TableMeta(storeType, options);
-
TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path);
+ if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) {
+ tableDesc.setExternal(true);
+ }
if (stats != null) {
tableDesc.setStats(stats);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
index 5c416c6..2fff67c 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
@@ -14,7 +14,6 @@
package org.apache.tajo.catalog.store;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
@@ -43,7 +42,7 @@ public class HCatalogStoreClientPool {
*/
public class HCatalogStoreClient {
private final HiveMetaStoreClient hiveClient;
- private boolean isInUse;
+ public AtomicBoolean isInUse = new AtomicBoolean(false);
private HCatalogStoreClient(HiveConf hiveConf) {
try {
@@ -54,7 +53,6 @@ public class HCatalogStoreClientPool {
// Turn in to an unchecked exception
throw new IllegalStateException(e);
}
- this.isInUse = false;
}
/**
@@ -68,24 +66,23 @@ public class HCatalogStoreClientPool {
* Returns this client back to the connection pool. If the connection pool has been
* closed, just close the Hive client connection.
*/
- public void release() {
- Preconditions.checkState(isInUse);
- isInUse = false;
+ public synchronized void release() {
+ if(!this.isInUse.getAndSet(false)){
+ return;
+ }
// Ensure the connection isn't returned to the pool if the pool has been closed.
// This lock is needed to ensure proper behavior when a thread reads poolClosed
// is false, but a call to pool.close() comes in immediately afterward.
- synchronized (poolClosed) {
- if (poolClosed.get()) {
- this.getHiveClient().close();
- } else {
- clientPool.add(this);
- }
+ if (poolClosed.get()) {
+ this.getHiveClient().close();
+ } else {
+ clientPool.add(this);
}
}
// Marks this client as in use
private void markInUse() {
- isInUse = true;
+ isInUse.set(true);
}
}
@@ -123,7 +120,7 @@ public class HCatalogStoreClientPool {
/**
* Gets a client from the pool. If the pool is empty a new client is created.
*/
- public HCatalogStoreClient getClient() {
+ public synchronized HCatalogStoreClient getClient() {
// The MetaStoreClient c'tor relies on knowing the Hadoop version by asking
// org.apache.hadoop.util.VersionInfo. The VersionInfo class relies on opening
// the 'common-version-info.properties' file as a resource from hadoop-common*.jar
@@ -149,11 +146,8 @@ public class HCatalogStoreClientPool {
*/
public void close() {
// Ensure no more items get added to the pool once close is called.
- synchronized (poolClosed) {
- if (poolClosed.get()) {
- return;
- }
- poolClosed.set(true);
+ if (poolClosed.getAndSet(true)) {
+ return;
}
HCatalogStoreClient client = null;
http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
index 396c41e..4d1cee1 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
@@ -111,7 +111,7 @@ public class KeyValueSet implements ProtoObject<KeyValueSetProto>, Cloneable, Gs
if(object instanceof KeyValueSet) {
KeyValueSet other = (KeyValueSet)object;
for(Entry<String, String> entry : other.keyVals.entrySet()) {
- if(!keyVals.get(entry.getKey()).equals(entry.getValue()))
+ if(!keyVals.containsKey(entry.getKey()) || !keyVals.get(entry.getKey()).equals(entry.getValue()))
return false;
}
return true;
http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
index 3d90a79..cb77936 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
@@ -441,71 +441,139 @@ public class TestCreateTable extends QueryTestCaseBase {
@Test
public final void testCreateTableLike1() throws Exception {
- // Basic create table with default database
- executeString("CREATE TABLE table1 (c1 int, c2 varchar);").close();
- executeString("CREATE TABLE table2 LIKE table1");
- String testMsg = "testCreateTableLike1: Basic create table with default db";
- assertTrue(testMsg,isClonedTable("table1","table2"));
- executeString("DROP TABLE table1");
- executeString("DROP TABLE table2");
-
- // Basic create table with database
- executeString("CREATE DATABASE d1").close();
- executeString("CREATE TABLE d1.table1 (c1 int, c2 varchar);").close();
- executeString("CREATE TABLE d1.table2 LIKE d1.table1");
- testMsg = "testCreateTableLike1: Basic create table with db test failed";
- assertTrue(testMsg, isClonedTable("d1.table1","d1.table2"));
- executeString("DROP TABLE d1.table1");
- executeString("DROP TABLE d1.table2");
-
- // Table with non-default store type
- executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING rcfile;").close();
- executeString("CREATE TABLE table2 LIKE table1");
- testMsg = "testCreateTableLike1: Table with non-default store type test failed";
- assertTrue(testMsg, isClonedTable("table1","table2"));
- executeString("DROP TABLE table1");
- executeString("DROP TABLE table2");
-
- // Table with non-default meta options
- executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close();
- executeString("CREATE TABLE table2 LIKE table1");
- testMsg = "testCreateTableLike1: Table with non-default meta options test failed";
- assertTrue(testMsg, isClonedTable("table1","table2"));
- executeString("DROP TABLE table1");
- executeString("DROP TABLE table2");
-
-
- // Table with partitions (default partition type)
- executeString("CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int, c4 float, c5 text);").close();
- executeString("CREATE TABLE table2 LIKE table1");
- testMsg = "testCreateTableLike1: Table with partitions test failed";
- assertTrue(testMsg, isClonedTable("table1","table2"));
- executeString("DROP TABLE table1");
- executeString("DROP TABLE table2");
-
-
- // Table with external flag
- // Use existing file as input for creating external table
- String className = getClass().getSimpleName();
- Path currentDatasetPath = new Path(datasetBasePath, className);
- Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
- executeString("CREATE EXTERNAL TABLE table3 (c1 int, c2 varchar) USING rcfile LOCATION '" + filePath.toUri() + "'").close();
- executeString("CREATE TABLE table2 LIKE table3");
- testMsg = "testCreateTableLike1: Table with external table flag test failed";
- assertTrue(testMsg, isClonedTable("table3","table2"));
- executeString("DROP TABLE table3");
- executeString("DROP TABLE table2");
-
-
- // Table created using CTAS
- executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close();
- executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2 as c2_a,c3 from table3;").close();
- executeString("CREATE TABLE table2 LIKE table4");
- testMsg = "testCreateTableLike1: Table using CTAS test failed";
- assertTrue(testMsg, isClonedTable("table4","table2"));
- executeString("DROP TABLE table3");
- executeString("DROP TABLE table4");
- executeString("DROP TABLE table2");
+ // Hcatalog does not support varchar type in hive-0.12.0
+ if (testingCluster.isHCatalogStoreRunning()) {
+ // Basic create table with default database
+ executeString("CREATE TABLE table1 (c1 int, c2 text);").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ String testMsg = "testCreateTableLike1: Basic create table with default db";
+ assertTrue(testMsg,isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+
+ // Basic create table with database
+ executeString("CREATE DATABASE d1").close();
+ executeString("CREATE TABLE d1.table1 (c1 int, c2 text);").close();
+ executeString("CREATE TABLE d1.table2 LIKE d1.table1");
+ testMsg = "testCreateTableLike1: Basic create table with db test failed";
+ assertTrue(testMsg, isClonedTable("d1.table1","d1.table2"));
+ executeString("DROP TABLE d1.table1");
+ executeString("DROP TABLE d1.table2");
+
+ // Table with non-default store type
+ executeString("CREATE TABLE table1 (c1 int, c2 text) USING rcfile;").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ testMsg = "testCreateTableLike1: Table with non-default store type test failed";
+ assertTrue(testMsg, isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+
+ // Table with non-default meta options
+ executeString("CREATE TABLE table1 (c1 int, c2 text) USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ testMsg = "testCreateTableLike1: Table with non-default meta options test failed";
+ assertTrue(testMsg, isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+
+
+ // Table with partitions (default partition type)
+ executeString("CREATE TABLE table1 (c1 int, c2 text) PARTITION BY COLUMN (c3 int, c4 float, c5 text);").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ testMsg = "testCreateTableLike1: Table with partitions test failed";
+ assertTrue(testMsg, isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+
+
+ // Table with external flag
+ // Use existing file as input for creating external table
+ String className = getClass().getSimpleName();
+ Path currentDatasetPath = new Path(datasetBasePath, className);
+ Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
+ executeString("CREATE EXTERNAL TABLE table3 (c1 int, c2 text) USING rcfile LOCATION '" + filePath.toUri() + "'").close();
+ executeString("CREATE TABLE table2 LIKE table3");
+ testMsg = "testCreateTableLike1: Table with external table flag test failed";
+ assertTrue(testMsg, isClonedTable("table3","table2"));
+ executeString("DROP TABLE table3");
+ executeString("DROP TABLE table2");
+
+
+ // Table created using CTAS
+ executeString("CREATE TABLE table3 (c1 int, c2 text) PARTITION BY COLUMN (c3 int);").close();
+ executeString("CREATE TABLE table4 AS SELECT c1 * c1 as m_c1, c2, c2 as c2_a,c3 from table3;").close();
+ executeString("CREATE TABLE table2 LIKE table4");
+ testMsg = "testCreateTableLike1: Table using CTAS test failed";
+ assertTrue(testMsg, isClonedTable("table4","table2"));
+ executeString("DROP TABLE table3");
+ executeString("DROP TABLE table4");
+ executeString("DROP TABLE table2");
+ } else {
+ // Basic create table with default database
+ executeString("CREATE TABLE table1 (c1 int, c2 varchar);").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ String testMsg = "testCreateTableLike1: Basic create table with default db";
+ assertTrue(testMsg,isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+
+ // Basic create table with database
+ executeString("CREATE DATABASE d1").close();
+ executeString("CREATE TABLE d1.table1 (c1 int, c2 varchar);").close();
+ executeString("CREATE TABLE d1.table2 LIKE d1.table1");
+ testMsg = "testCreateTableLike1: Basic create table with db test failed";
+ assertTrue(testMsg, isClonedTable("d1.table1","d1.table2"));
+ executeString("DROP TABLE d1.table1");
+ executeString("DROP TABLE d1.table2");
+
+ // Table with non-default store type
+ executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING rcfile;").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ testMsg = "testCreateTableLike1: Table with non-default store type test failed";
+ assertTrue(testMsg, isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+
+ // Table with non-default meta options
+ executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ testMsg = "testCreateTableLike1: Table with non-default meta options test failed";
+ assertTrue(testMsg, isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+
+
+ // Table with partitions (default partition type)
+ executeString("CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int, c4 float, c5 text);").close();
+ executeString("CREATE TABLE table2 LIKE table1");
+ testMsg = "testCreateTableLike1: Table with partitions test failed";
+ assertTrue(testMsg, isClonedTable("table1","table2"));
+ executeString("DROP TABLE table1");
+ executeString("DROP TABLE table2");
+
+
+ // Table with external flag
+ // Use existing file as input for creating external table
+ String className = getClass().getSimpleName();
+ Path currentDatasetPath = new Path(datasetBasePath, className);
+ Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
+ executeString("CREATE EXTERNAL TABLE table3 (c1 int, c2 varchar) USING rcfile LOCATION '" + filePath.toUri() + "'").close();
+ executeString("CREATE TABLE table2 LIKE table3");
+ testMsg = "testCreateTableLike1: Table with external table flag test failed";
+ assertTrue(testMsg, isClonedTable("table3","table2"));
+ executeString("DROP TABLE table3");
+ executeString("DROP TABLE table2");
+
+
+ // Table created using CTAS
+ executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close();
+ executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2 as c2_a,c3 from table3;").close();
+ executeString("CREATE TABLE table2 LIKE table4");
+ testMsg = "testCreateTableLike1: Table using CTAS test failed";
+ assertTrue(testMsg, isClonedTable("table4","table2"));
+ executeString("DROP TABLE table3");
+ executeString("DROP TABLE table4");
+ executeString("DROP TABLE table2");
/* Enable when view is supported
@@ -529,5 +597,6 @@ public class TestCreateTable extends QueryTestCaseBase {
executeString("DROP TABLE table1");
executeString("DROP TABLE table2");
*/
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
index fdbca61..7ca94c9 100644
--- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
+++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
@@ -129,6 +129,9 @@ public class TestResultSet {
@Test
public void testDateTimeType() throws Exception {
+ // Hcatalog does not support date type, time type in hive-0.12.0
+ if(util.isHCatalogStoreRunning()) return;
+
TimeZone tajoCurrentTimeZone = TajoConf.getCurrentTimeZone();
TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("UTC"));
http://git-wip-us.apache.org/repos/asf/tajo/blob/a17bd0f1/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java b/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java
index 0e2db45..72223c1 100644
--- a/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java
+++ b/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java
@@ -135,15 +135,21 @@ public class TestAsyncRpc {
assertTrue(future.isDone());
}
- @Test(expected = TimeoutException.class)
+ @Test
public void testCallFutureTimeout() throws Exception {
- EchoMessage echoMessage = EchoMessage.newBuilder()
- .setMessage(MESSAGE).build();
- CallFuture<EchoMessage> future = new CallFuture<EchoMessage>();
- stub.deley(null, echoMessage, future);
-
- assertFalse(future.isDone());
- assertEquals(future.get(1, TimeUnit.SECONDS), echoMessage);
+ boolean timeout = false;
+ try {
+ EchoMessage echoMessage = EchoMessage.newBuilder()
+ .setMessage(MESSAGE).build();
+ CallFuture<EchoMessage> future = new CallFuture<EchoMessage>();
+ stub.deley(null, echoMessage, future);
+
+ assertFalse(future.isDone());
+ future.get(1, TimeUnit.SECONDS);
+ } catch (TimeoutException te) {
+ timeout = true;
+ }
+ assertTrue(timeout);
}
@Test