You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by jh...@apache.org on 2014/08/11 08:19:34 UTC
git commit: TAJO-995: HiveMetaStoreClient wrapper should retry the
connection. (jinho)
Repository: tajo
Updated Branches:
refs/heads/master 87e7ba214 -> fbf716689
TAJO-995: HiveMetaStoreClient wrapper should retry the connection. (jinho)
Closes #108
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fbf71668
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fbf71668
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fbf71668
Branch: refs/heads/master
Commit: fbf7166897b3b25d8319198216dc4ca529e677ce
Parents: 87e7ba2
Author: jinossy <ji...@gmail.com>
Authored: Mon Aug 11 15:18:29 2014 +0900
Committer: jinossy <ji...@gmail.com>
Committed: Mon Aug 11 15:18:29 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
.../tajo/catalog/store/HCatalogStore.java | 11 ++++++-----
.../catalog/store/HCatalogStoreClientPool.java | 20 ++++++++++++++++----
.../apache/tajo/catalog/store/HCatalogUtil.java | 6 ++++++
4 files changed, 30 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fbf71668/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8788b17..839125e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -109,6 +109,8 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-995: HiveMetaStoreClient wrapper should retry the connection. (jinho)
+
TAJO-947: ColPartitionStoreExec can cause URISyntaxException due
to special characters. (Mai Hai Thanh via hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/fbf71668/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 61bdccd..fb8f4c4 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -48,6 +48,7 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.InternalException;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.KeyValueSet;
+import org.apache.thrift.TException;
import java.io.IOException;
import java.util.*;
@@ -77,13 +78,13 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
@Override
public boolean existTable(final String databaseName, final String tableName) throws CatalogException {
boolean exist = false;
- org.apache.hadoop.hive.ql.metadata.Table table = null;
+ org.apache.hadoop.hive.ql.metadata.Table table;
HCatalogStoreClientPool.HCatalogStoreClient client = null;
// get table
try {
client = clientPool.getClient();
- table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName);
+ table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName);
if (table != null) {
exist = true;
}
@@ -118,7 +119,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
// get hive table schema
try {
client = clientPool.getClient();
- table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName);
+ table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName);
path = table.getPath();
} catch (NoSuchObjectException nsoe) {
throw new CatalogException("Table not found. - tableName:" + tableName, nsoe);
@@ -291,7 +292,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
try {
client = clientPool.getClient();
return client.getHiveClient().getAllTables(databaseName);
- } catch (MetaException e) {
+ } catch (TException e) {
throw new CatalogException(e);
} finally {
if(client != null) client.release();
@@ -401,7 +402,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
try {
client = clientPool.getClient();
return client.getHiveClient().getAllDatabases();
- } catch (MetaException e) {
+ } catch (TException e) {
throw new CatalogException(e);
} finally {
if (client != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/fbf71668/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
index 2fff67c..8ccb100 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
@@ -16,7 +16,9 @@ package org.apache.tajo.catalog.store;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.*;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.log4j.Logger;
import java.util.Iterator;
@@ -41,12 +43,22 @@ public class HCatalogStoreClientPool {
* connection pool.
*/
public class HCatalogStoreClient {
- private final HiveMetaStoreClient hiveClient;
+ private final IMetaStoreClient hiveClient;
public AtomicBoolean isInUse = new AtomicBoolean(false);
private HCatalogStoreClient(HiveConf hiveConf) {
try {
- this.hiveClient = new HiveMetaStoreClient(hiveConf);
+ HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
+ @Override
+ public HiveMetaHook getHook(Table table) throws MetaException {
+ /* metadata hook implementation, or null if this
+ * storage handler does not need any metadata notifications
+ */
+ return null;
+ }
+ };
+
+ this.hiveClient = RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName());
clientPool.add(this);
LOG.info("MetaStoreClient created (size = " + clientPool.size() + ")");
} catch (Exception e) {
@@ -58,7 +70,7 @@ public class HCatalogStoreClientPool {
/**
* Returns the internal HiveMetaStoreClient object.
*/
- public HiveMetaStoreClient getHiveClient() {
+ public IMetaStoreClient getHiveClient() {
return hiveClient;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/fbf71668/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
index 9e60768..54fdb9d 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
@@ -20,9 +20,11 @@ package org.apache.tajo.catalog.store;
import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hcatalog.common.HCatException;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
@@ -30,6 +32,7 @@ import org.apache.hcatalog.data.schema.HCatSchema;
import org.apache.tajo.catalog.exception.CatalogException;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes;
+import org.apache.thrift.TException;
import parquet.hadoop.mapred.DeprecatedParquetOutputFormat;
public class HCatalogUtil {
@@ -137,4 +140,7 @@ public class HCatalogUtil {
}
}
+ public static Table getTable(IMetaStoreClient client, String dbName, String tableName) throws TException {
+ return new Table(client.getTable(dbName, tableName));
+ }
}