You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by bl...@apache.org on 2014/01/07 10:15:52 UTC
git commit: TAJO-433: Improve integration with Hive. (jaehwa)
Updated Branches:
refs/heads/master 7f9890801 -> 19a756d75
TAJO-433: Improve integration with Hive. (jaehwa)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/19a756d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/19a756d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/19a756d7
Branch: refs/heads/master
Commit: 19a756d75ef693df4f28a42a9cd3634524f3f577
Parents: 7f98908
Author: blrunner <jh...@gruter.com>
Authored: Tue Jan 7 18:15:24 2014 +0900
Committer: blrunner <jh...@gruter.com>
Committed: Tue Jan 7 18:15:24 2014 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
NOTICE.txt | 3 +-
.../tajo-catalog-drivers/tajo-hcatalog/pom.xml | 170 +++++++++++++------
.../tajo/catalog/store/HCatalogStore.java | 71 ++++----
.../catalog/store/HCatalogStoreClientPool.java | 153 +++++++++++++++++
.../apache/tajo/catalog/store/HCatalogUtil.java | 29 ----
.../tajo/catalog/store/TestHCatalogStore.java | 31 +++-
.../org/apache/tajo/catalog/CatalogServer.java | 5 +
tajo-dist/src/main/bin/tajo | 38 ++++-
tajo-dist/src/main/conf/tajo-env.sh | 3 +-
10 files changed, 380 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/19a756d7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2422573..0253ddb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -110,6 +110,8 @@ Release 0.8.0 - unreleased
IMPROVEMENTS
+ TAJO-433: Improve integration with Hive. (jaehwa)
+
TAJO-471: Extract ColumnPartitonUtils class for ColumnPartition rewrite.
(DaeMyung Kang via hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/19a756d7/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index f643a4f..543bad4 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -6,4 +6,5 @@ The Apache Software Foundation (http://www.apache.org/).
This product includes Protobuf-java-format (http://code.google.com/p/protobuf-java-format/)
Copyright 2000-2011 NeuStar, Inc. All rights reserved.
-Copyright (c) 2009, Orbitz World Wide All rights reserved.
\ No newline at end of file
+Copyright (c) 2009, Orbitz World Wide All rights reserved.
+Copyright 2012 Cloudera Inc.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/19a756d7/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
index fb1b827..86ba343 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
@@ -116,6 +116,18 @@
<artifactId>derby</artifactId>
<version>10.8.2.2</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libfb303</artifactId>
+ <version>0.9.0</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <version>0.9.0</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<profiles>
@@ -132,66 +144,85 @@
<groupId>javax.jdo</groupId>
<artifactId>jdo2-api</artifactId>
<version>2.3-eb</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>${hive.version}</version>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-builtins</artifactId>
+ <artifactId>hive-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
+ <artifactId>hive-contrib</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
+ <artifactId>hive-hbase-handler</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-pdk</artifactId>
+ <artifactId>hive-metastore</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-service</artifactId>
+ <artifactId>hive-serde</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
<artifactId>hive-shims</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-testutils</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libfb303</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jdo</groupId>
+ <artifactId>jdo2-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-builtins</artifactId>
+ <artifactId>hive-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
+ <artifactId>hive-serde</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
+ <artifactId>hive-shimss</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-pdk</artifactId>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libfb303</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-service</artifactId>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shims</artifactId>
+ <groupId>javax.jdo</groupId>
+ <artifactId>jdo2-api</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -199,22 +230,23 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-cli</artifactId>
<version>${hive.version}</version>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-builtins</artifactId>
+ <artifactId>hive-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
+ <artifactId>hive-exec</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
+ <artifactId>hive-metastore</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-pdk</artifactId>
+ <artifactId>hive-serde</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
@@ -224,28 +256,37 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-shims</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.jdo</groupId>
+ <artifactId>jdo2-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hcatalog</groupId>
<artifactId>hcatalog-core</artifactId>
<version>${hive.version}</version>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-builtins</artifactId>
+ <artifactId>hive-cli</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
+ <artifactId>hive-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
+ <artifactId>hive-exec</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-pdk</artifactId>
+ <artifactId>hive-metastore</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-serde</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
@@ -255,6 +296,10 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-shims</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.jdo</groupId>
+ <artifactId>jdo2-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -275,69 +320,82 @@
</properties>
<dependencies>
<dependency>
- <groupId>javax.jdo</groupId>
- <artifactId>jdo2-api</artifactId>
- <version>2.3-eb</version>
- </dependency>
- <dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>${hive.version}</version>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-builtins</artifactId>
+ <artifactId>hive-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
+ <artifactId>hive-contrib</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
+ <artifactId>hive-hbase-handler</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-pdk</artifactId>
+ <artifactId>hive-metastore</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-service</artifactId>
+ <artifactId>hive-serde</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
<artifactId>hive-shims</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-testutils</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libfb303</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.jolbox</groupId>
+ <artifactId>bonecp</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-builtins</artifactId>
+ <artifactId>hive-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
+ <artifactId>hive-serde</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
+ <artifactId>hive-shimss</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-pdk</artifactId>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libfb303</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-service</artifactId>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shims</artifactId>
+ <groupId>com.jolbox</groupId>
+ <artifactId>bonecp</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -345,22 +403,23 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-cli</artifactId>
<version>${hive.version}</version>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-builtins</artifactId>
+ <artifactId>hive-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
+ <artifactId>hive-exec</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
+ <artifactId>hive-metastore</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-pdk</artifactId>
+ <artifactId>hive-serde</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
@@ -370,28 +429,37 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-shims</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.jolbox</groupId>
+ <artifactId>bonecp</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive.hcatalog</groupId>
<artifactId>hcatalog-core</artifactId>
<version>${hive.version}</version>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-builtins</artifactId>
+ <artifactId>hive-cli</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
+ <artifactId>hive-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
+ <artifactId>hive-exec</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-pdk</artifactId>
+ <artifactId>hive-metastore</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-serde</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
@@ -401,6 +469,10 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-shims</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.jolbox</groupId>
+ <artifactId>bonecp</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/19a756d7/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 72b2aff..8ba3790 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -21,8 +21,8 @@ package org.apache.tajo.catalog.store;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hcatalog.common.HCatUtil;
@@ -42,30 +42,29 @@ import java.util.*;
public class HCatalogStore extends CatalogConstants implements CatalogStore {
protected final Log LOG = LogFactory.getLog(getClass());
protected Configuration conf;
- protected String catalogUri;
+ private static final int CLIENT_POOL_SIZE = 5;
+ private final HCatalogStoreClientPool clientPool = new HCatalogStoreClientPool(0);
+
private Map<Pair<String, String>, Table> tableMap = new HashMap<Pair<String, String>, Table>();
public HCatalogStore(final Configuration conf)
throws InternalException {
this.conf = conf;
- if(conf.get(CatalogConstants.DEPRECATED_CATALOG_URI) != null) {
- LOG.warn("Configuration parameter " + CatalogConstants.DEPRECATED_CATALOG_URI + " " +
- "is deprecated. Use " + CatalogConstants.CATALOG_URI + " instead.");
- this.catalogUri = conf.get(CatalogConstants.DEPRECATED_CATALOG_URI);
- } else {
- this.catalogUri = conf.get(CatalogConstants.CATALOG_URI);
+ try {
+ clientPool.addClients(CLIENT_POOL_SIZE);
+ } catch (Exception e) {
+ e.printStackTrace();
}
}
@Override
- public final boolean existTable(final String name) throws IOException {
+ public boolean existTable(final String name) throws IOException {
boolean exist = false;
String dbName = null, tableName = null;
Pair<String, String> tablePair = null;
org.apache.hadoop.hive.ql.metadata.Table table = null;
- HiveMetaStoreClient client = null;
-
+ HCatalogStoreClientPool.HCatalogStoreClient client = null;
// get db name and table name.
try {
tablePair = HCatUtil.getDbAndTableName(name);
@@ -78,8 +77,8 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
// get table
try {
try {
- client = HCatalogUtil.getHiveMetaClient(catalogUri, null);
- table = HCatUtil.getTable(client, dbName, tableName);
+ client = clientPool.getClient();
+ table = HCatUtil.getTable(client.getHiveClient(), dbName, tableName);
if (table != null) {
exist = true;
}
@@ -89,7 +88,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
throw new IOException(e);
}
} finally {
- HCatUtil.closeHiveClientQuietly(client);
+ client.release();
}
return exist;
@@ -100,7 +99,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
String dbName = null, tableName = null;
Pair<String, String> tablePair = null;
org.apache.hadoop.hive.ql.metadata.Table table = null;
- HiveMetaStoreClient client = null;
+ HCatalogStoreClientPool.HCatalogStoreClient client = null;
Path path = null;
CatalogProtos.StoreType storeType = null;
org.apache.tajo.catalog.Schema schema = null;
@@ -123,8 +122,8 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
try {
// get hive table schema
try {
- client = HCatalogUtil.getHiveMetaClient(catalogUri, null);
- table = HCatUtil.getTable(client, dbName, tableName);
+ client = clientPool.getClient();
+ table = HCatUtil.getTable(client.getHiveClient(), dbName, tableName);
path = table.getPath();
} catch (NoSuchObjectException nsoe) {
throw new InternalException("Table not found. - tableName:" + name, nsoe);
@@ -170,9 +169,16 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
// set data size
+ long totalSize = 0;
if(properties.getProperty("totalSize") != null) {
- stats.setNumBytes(new Long(properties.getProperty("totalSize")));
+ totalSize = new Long(properties.getProperty("totalSize"));
+ } else {
+ FileSystem fs = path.getFileSystem(conf);
+ if (fs.exists(path)) {
+ totalSize = fs.getContentSummary(path).getLength();
+ }
}
+ stats.setNumBytes(totalSize);
}
// set partition keys
@@ -189,7 +195,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
}
} finally {
- HCatUtil.closeHiveClientQuietly(client);
+ client.release();
}
TableMeta meta = new TableMeta(storeType, options);
@@ -218,14 +224,14 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
List<String> dbs = null;
List<String> tables = null;
List<String> allTables = new ArrayList<String>();
- HiveMetaStoreClient client = null;
+ HCatalogStoreClientPool.HCatalogStoreClient client = null;
try {
try {
- client = HCatalogUtil.getHiveMetaClient(catalogUri, null);
- dbs = client.getAllDatabases();
+ client = clientPool.getClient();
+ dbs = client.getHiveClient().getAllDatabases();
for(String eachDB: dbs) {
- tables = client.getAllTables(eachDB);
+ tables = client.getHiveClient().getAllTables(eachDB);
for(String eachTable: tables) {
allTables.add(eachDB + "." + eachTable);
}
@@ -235,7 +241,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
} finally {
- HCatUtil.closeHiveClientQuietly(client);
+ client.release();
}
return allTables;
}
@@ -244,7 +250,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
public final void addTable(final TableDesc tableDesc) throws IOException {
String dbName = null, tableName = null;
Pair<String, String> tablePair = null;
- HiveMetaStoreClient client = null;
+ HCatalogStoreClientPool.HCatalogStoreClient client = null;
// get db name and table name.
try {
@@ -257,7 +263,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
try {
try {
- client = HCatalogUtil.getHiveMetaClient(catalogUri, null);
+ client = clientPool.getClient();
org.apache.hadoop.hive.metastore.api.Table table = new org.apache.hadoop.hive.metastore.api
.Table();
@@ -310,12 +316,12 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
sd.setSortCols(new ArrayList<Order>());
table.setSd(sd);
- client.createTable(table);
+ client.getHiveClient().createTable(table);
} catch (Exception e) {
throw new IOException(e);
}
} finally {
- HCatUtil.closeHiveClientQuietly(client);
+ client.release();
}
}
@@ -323,7 +329,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
public final void deleteTable(final String name) throws IOException {
String dbName = null, tableName = null;
Pair<String, String> tablePair = null;
- HiveMetaStoreClient client = null;
+ HCatalogStoreClientPool.HCatalogStoreClient client = null;
// get db name and table name.
try {
@@ -335,13 +341,13 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
try {
- client = HCatalogUtil.getHiveMetaClient(catalogUri, null);
- client.dropTable(dbName, tableName, false, false);
+ client = clientPool.getClient();
+ client.getHiveClient().dropTable(dbName, tableName, false, false);
} catch (NoSuchObjectException nsoe) {
} catch (Exception e) {
throw new IOException(e);
} finally {
- HCatUtil.closeHiveClientQuietly(client);
+ client.release();
}
}
@Override
@@ -408,5 +414,6 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
@Override
public final void close() {
+ clientPool.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/19a756d7/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
new file mode 100644
index 0000000..263e1dd
--- /dev/null
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java
@@ -0,0 +1,153 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package org.apache.tajo.catalog.store;
+
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.log4j.Logger;
+
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+/**
+ * Manages a pool of HiveMetaStoreClient connections. If the connection pool is empty
+ * a new client is created and added to the pool. There is no size limit.
+ */
+public class HCatalogStoreClientPool {
+ private static final Logger LOG = Logger.getLogger(HCatalogStoreClientPool.class);
+ private final ConcurrentLinkedQueue<HCatalogStoreClient> clientPool =
+ new ConcurrentLinkedQueue<HCatalogStoreClient>();
+ private Boolean poolClosed = false;
+ private final HiveConf hiveConf;
+
+ /**
+ * A wrapper around the HiveMetaStoreClient that manages interactions with the
+ * connection pool.
+ */
+ public class HCatalogStoreClient {
+ private final HiveMetaStoreClient hiveClient;
+ private boolean isInUse;
+
+ private HCatalogStoreClient(HiveConf hiveConf) {
+ try {
+ LOG.debug("Creating MetaStoreClient. Pool Size = " + clientPool.size());
+ this.hiveClient = new HiveMetaStoreClient(hiveConf);
+ } catch (Exception e) {
+ // Turn in to an unchecked exception
+ throw new IllegalStateException(e);
+ }
+ this.isInUse = false;
+ }
+
+ /**
+ * Returns the internal HiveMetaStoreClient object.
+ */
+ public HiveMetaStoreClient getHiveClient() {
+ return hiveClient;
+ }
+
+ /**
+ * Returns this client back to the connection pool. If the connection pool has been
+ * closed, just close the Hive client connection.
+ */
+ public void release() {
+ Preconditions.checkState(isInUse);
+ isInUse = false;
+ // Ensure the connection isn't returned to the pool if the pool has been closed.
+ // This lock is needed to ensure proper behavior when a thread reads poolClosed
+ // is false, but a call to pool.close() comes in immediately afterward.
+ synchronized (poolClosed) {
+ if (poolClosed) {
+ hiveClient.close();
+ } else {
+ // TODO: Currently the pool does not work properly because we cannot
+ // reuse MetastoreClient connections. No reason to add this client back
+ // to the pool. See HIVE-5181.
+ // clientPool.add(this);
+ hiveClient.close();
+ }
+ }
+ }
+
+ // Marks this client as in use
+ private void markInUse() {
+ isInUse = true;
+ }
+ }
+
+ public HCatalogStoreClientPool(int initialSize) {
+ this(initialSize, new HiveConf(HCatalogStoreClientPool.class));
+ }
+
+ public HCatalogStoreClientPool(int initialSize, HiveConf hiveConf) {
+ this.hiveConf = hiveConf;
+ addClients(initialSize);
+ }
+
+ /**
+ * Add numClients to the client pool.
+ */
+ public void addClients(int numClients) {
+ for (int i = 0; i < numClients; ++i) {
+ clientPool.add(new HCatalogStoreClient(hiveConf));
+ }
+ }
+
+ /**
+ * Gets a client from the pool. If the pool is empty a new client is created.
+ */
+ public HCatalogStoreClient getClient() {
+ // The MetaStoreClient c'tor relies on knowing the Hadoop version by asking
+ // org.apache.hadoop.util.VersionInfo. The VersionInfo class relies on opening
+ // the 'common-version-info.properties' file as a resource from hadoop-common*.jar
+ // using the Thread's context classloader. If necessary, set the Thread's context
+ // classloader, otherwise VersionInfo will fail in it's c'tor.
+ if (Thread.currentThread().getContextClassLoader() == null) {
+ Thread.currentThread().setContextClassLoader(ClassLoader.getSystemClassLoader());
+ }
+
+ HCatalogStoreClient client = clientPool.poll();
+ // The pool was empty so create a new client and return that.
+ if (client == null) {
+ client = new HCatalogStoreClient(hiveConf);
+ } else {
+ // TODO: Due to Hive Metastore bugs, there is leftover state from previous client
+ // connections so we are unable to reuse the same connection. For now simply
+ // reconnect each time. One possible culprit is HIVE-5181.
+ client = new HCatalogStoreClient(hiveConf);
+ }
+ client.markInUse();
+ return client;
+ }
+
+ /**
+ * Removes all items from the connection pool and closes all Hive Meta Store client
+ * connections. Can be called multiple times.
+ */
+ public void close() {
+ // Ensure no more items get added to the pool once close is called.
+ synchronized (poolClosed) {
+ if (poolClosed) {
+ return;
+ }
+ poolClosed = true;
+ }
+
+ HCatalogStoreClient client = null;
+ while ((client = clientPool.poll()) != null) {
+ client.getHiveClient().close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/19a756d7/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
index b92cbf2..33b3e45 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
@@ -19,11 +19,8 @@ package org.apache.tajo.catalog.store;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hcatalog.common.HCatException;
-import org.apache.hcatalog.common.HCatUtil;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hcatalog.data.schema.HCatSchema;
import org.apache.tajo.catalog.proto.CatalogProtos;
@@ -59,32 +56,6 @@ public class HCatalogUtil {
}
}
- public static HiveMetaStoreClient getHiveMetaClient(String metaStoreUri,
- String metaStoreKerberosPrincipal)
- //Class<?> cls)
- throws Exception {
-// HiveConf hiveConf = new HiveConf(cls);
-
- HiveConf hiveConf = new HiveConf();
-
- if (metaStoreUri != null) {
- hiveConf.set("hive.metastore.local", "false");
- hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreUri.trim());
- }
-
- if (metaStoreKerberosPrincipal != null) {
- hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, true);
- hiveConf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL, metaStoreKerberosPrincipal);
- }
-
- try {
- return HCatUtil.getHiveClient(hiveConf);
- } catch (Exception e) {
- throw new InternalException("Tajo cannot connect Hive metastore. - serverUri:" +
- metaStoreUri, e);
- }
- }
-
public static TajoDataTypes.Type getTajoFieldType(String fieldType) throws IOException {
if(fieldType == null) {
throw new InternalException("Hive field type is null.");
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/19a756d7/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
index fb2abea..0d521c8 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
@@ -61,6 +61,9 @@ public class TestHCatalogStore {
@BeforeClass
public static void setUp() throws Exception {
+ // delete metstore default path for successful unit tests
+ deleteMetaStoreDirectory();
+
// Set Hive MetaStore
Database db = new Database();
db.setName(DB_NAME);
@@ -80,10 +83,18 @@ public class TestHCatalogStore {
conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
conf.set(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname, DummyListener.class.getName());
+ // set property to use at HCatalogUtil
+ System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname, metastoreUri);
+ System.setProperty(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES.varname, "3");
+ System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
+ System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
+ System.setProperty(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+ System.setProperty(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname, DummyListener.class.getName());
+
SessionState.start(new CliSessionState(conf));
// create database and tables on Hive MetaStore.
- client = new HiveMetaStoreClient(conf, null);
+ client = new HiveMetaStoreClient(conf);
client.createDatabase(db);
createTable(NATION);
@@ -93,13 +104,21 @@ public class TestHCatalogStore {
// create local HCatalogStore.
TajoConf tajoConf = new TajoConf();
- tajoConf.set(CatalogConstants.CATALOG_URI, metastoreUri);
tajoConf.set(CatalogConstants.STORE_CLASS, HCatalogStore.class.getCanonicalName());
tajoConf.setVar(TajoConf.ConfVars.CATALOG_ADDRESS, "127.0.0.1:0");
store = new HCatalogStore(tajoConf);
}
+ private static void deleteMetaStoreDirectory() throws Exception {
+ Path path = new Path("metastore_db");
+ FileSystem fs = FileSystem.getLocal(new Configuration());
+ if(fs.exists(path)) {
+ fs.delete(path, true);
+ }
+ fs.close();
+ }
+
private static void createTable(String tableName) throws Exception {
Map<String, String> tableParams = new HashMap<String, String>();
@@ -164,13 +183,7 @@ public class TestHCatalogStore {
private static void dropDatabase() throws Exception {
try {
client.dropDatabase(DB_NAME);
-
- Path path = new Path("metastore_db");
- FileSystem fs = FileSystem.getLocal(new Configuration());
- if(fs.exists(path)) {
- fs.delete(path, true);
- }
- fs.close();
+ deleteMetaStoreDirectory();
} catch (NoSuchObjectException e) {
} catch (InvalidOperationException e) {
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/19a756d7/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
index 6c16558..d14a3a9 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
@@ -171,6 +171,11 @@ public class CatalogServer extends AbstractService {
this.rpcServer.shutdown();
}
LOG.info("Catalog Server (" + bindAddressStr + ") shutdown");
+ try {
+ store.close();
+ } catch (IOException ioe) {
+ LOG.error(ioe);
+ }
super.stop();
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/19a756d7/tajo-dist/src/main/bin/tajo
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo
index f31c701..d1682ce 100755
--- a/tajo-dist/src/main/bin/tajo
+++ b/tajo-dist/src/main/bin/tajo
@@ -212,6 +212,13 @@ for f in $TAJO_CONF_DIR/*.xml; do
TAJO_BASE_CLASSPATH=${TAJO_BASE_CLASSPATH}:$f;
done
+#HIVE_CONF=$HIVE_HOME/conf
+#if [ -d ${HIVE_CONF} ]; then
+# for f in ${HIVE_CONF}/hive-site.xml; do
+# TAJO_BASE_CLASSPATH=${TAJO_BASE_CLASSPATH}:$f;
+# done
+#fi
+
# Tajo Jar Directory
TAJO_JAR_DIRS="$TAJO_HOME
${TAJO_HOME}/lib"
@@ -235,24 +242,47 @@ CLASSPATH="${CLASSPATH}:${TAJO_BASE_CLASSPATH}"
# Find and Set Hive CLASSPATH
##############################################################################
+HIVE_CONF=$HIVE_HOME/conf
+
+if [ -d ${HIVE_CONF} ]; then
+ CLASSPATH=${CLASSPATH}:${HIVE_CONF}
+
+ for f in ${HIVE_CONF}/hive-site.xml; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+fi
+
HIVE_LIB=$HIVE_HOME/lib
if [ -d ${HIVE_LIB} ]; then
- for f in ${HIVE_LIB}/*.jar; do
+
+ for f in ${HIVE_LIB}/hive-*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
- for f in $HIVE_HOME/hcatalog/share/hcatalog/*.jar; do
+ for f in ${HIVE_LIB}/libfb303-*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
- for f in $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-core-*.jar; do
+ for f in ${HIVE_LIB}/jdo-api-*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
- for f in $HADOOP_HOME/hadoop-core-*.jar; do
+ for f in ${HIVE_LIB}/datanucleus-*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
+
+ for f in $HIVE_HOME/hcatalog/share/hcatalog/hcatalog-core-*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+
+ for f in $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-core-*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+fi
+
+if [ ${HIVE_JDBC_DRIVER_DIR} != "" ]; then
+ CLASSPATH=${CLASSPATH}:$HIVE_JDBC_DRIVER_DIR;
fi
##############################################################################
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/19a756d7/tajo-dist/src/main/conf/tajo-env.sh
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/conf/tajo-env.sh b/tajo-dist/src/main/conf/tajo-env.sh
index bb5cf4b..e61415c 100755
--- a/tajo-dist/src/main/conf/tajo-env.sh
+++ b/tajo-dist/src/main/conf/tajo-env.sh
@@ -67,4 +67,5 @@
export TAJO_WORKER_STANDBY_MODE=true
# It must be required to use HCatalogStore
-# export HIVE_HOME=
\ No newline at end of file
+# export HIVE_HOME=
+# export HIVE_JDBC_DRIVER_DIR=
\ No newline at end of file