You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2017/11/02 16:23:14 UTC
[12/12] hive git commit: HIVE-17812 Move remaining classes that
HiveMetaStore depends on. This closes #261. (Alan Gates,
reviewed by Vihang Karajgaonkar)
HIVE-17812 Move remaining classes that HiveMetaStore depends on. This closes #261. (Alan Gates, reviewed by Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c5a9673a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c5a9673a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c5a9673a
Branch: refs/heads/master
Commit: c5a9673a014ad7362a3cbd8a53fe021ee7b7082f
Parents: 10aa330
Author: Alan Gates <ga...@hortonworks.com>
Authored: Thu Nov 2 09:21:25 2017 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Thu Nov 2 09:21:25 2017 -0700
----------------------------------------------------------------------
.../hcatalog/listener/NotificationListener.java | 24 +-
.../hive/hcatalog/streaming/TestStreaming.java | 16 +-
.../metastore/TestMetaStoreEventListener.java | 4 +-
.../hive/metastore/TestMetaStoreUtils.java | 62 --
...DummyHiveMetastoreAuthorizationProvider.java | 4 +-
.../security/InjectableDummyAuthenticator.java | 4 +-
.../hive/metastore/AcidEventListener.java | 94 --
.../hadoop/hive/metastore/AlterHandler.java | 196 ----
.../hive/metastore/FileMetadataManager.java | 134 ---
.../hive/metastore/HMSMetricsListener.java | 90 --
.../hadoop/hive/metastore/HiveAlterHandler.java | 900 -------------------
.../hadoop/hive/metastore/HiveMetaStore.java | 91 +-
.../hive/metastore/HouseKeeperService.java | 45 -
.../hive/metastore/MetaStoreEventListener.java | 227 -----
.../MetaStoreEventListenerConstants.java | 41 -
.../hive/metastore/MetaStoreInitContext.java | 27 -
.../hive/metastore/MetaStoreInitListener.java | 49 -
.../metastore/MetaStoreListenerNotifier.java | 266 ------
.../metastore/MetaStorePreEventListener.java | 57 --
.../hadoop/hive/metastore/MetaStoreUtils.java | 189 +---
.../hive/metastore/RetryingHMSHandler.java | 233 -----
.../metastore/SessionPropertiesListener.java | 45 -
.../TransactionalMetaStoreEventListener.java | 39 -
.../TransactionalValidationListener.java | 359 --------
.../metastore/events/AddForeignKeyEvent.java | 37 -
.../hive/metastore/events/AddIndexEvent.java | 40 -
.../events/AddNotNullConstraintEvent.java | 37 -
.../metastore/events/AddPartitionEvent.java | 82 --
.../metastore/events/AddPrimaryKeyEvent.java | 38 -
.../events/AddUniqueConstraintEvent.java | 37 -
.../hive/metastore/events/AlterIndexEvent.java | 47 -
.../metastore/events/AlterPartitionEvent.java | 75 --
.../hive/metastore/events/AlterTableEvent.java | 62 --
.../metastore/events/ConfigChangeEvent.java | 52 --
.../metastore/events/CreateDatabaseEvent.java | 43 -
.../metastore/events/CreateFunctionEvent.java | 43 -
.../hive/metastore/events/CreateTableEvent.java | 43 -
.../metastore/events/DropConstraintEvent.java | 47 -
.../metastore/events/DropDatabaseEvent.java | 43 -
.../metastore/events/DropFunctionEvent.java | 43 -
.../hive/metastore/events/DropIndexEvent.java | 40 -
.../metastore/events/DropPartitionEvent.java | 70 --
.../hive/metastore/events/DropTableEvent.java | 54 --
.../hive/metastore/events/EventCleanerTask.java | 52 --
.../hive/metastore/events/InsertEvent.java | 114 ---
.../hive/metastore/events/ListenerEvent.java | 178 ----
.../events/LoadPartitionDoneEvent.java | 57 --
.../hive/metastore/events/PreAddIndexEvent.java | 41 -
.../metastore/events/PreAddPartitionEvent.java | 79 --
.../metastore/events/PreAlterDatabaseEvent.java | 47 -
.../metastore/events/PreAlterIndexEvent.java | 47 -
.../events/PreAlterPartitionEvent.java | 65 --
.../metastore/events/PreAlterTableEvent.java | 53 --
.../events/PreAuthorizationCallEvent.java | 33 -
.../events/PreCreateDatabaseEvent.java | 43 -
.../metastore/events/PreCreateTableEvent.java | 43 -
.../metastore/events/PreDropDatabaseEvent.java | 43 -
.../metastore/events/PreDropIndexEvent.java | 40 -
.../metastore/events/PreDropPartitionEvent.java | 67 --
.../metastore/events/PreDropTableEvent.java | 55 --
.../hive/metastore/events/PreEventContext.java | 74 --
.../events/PreLoadPartitionDoneEvent.java | 58 --
.../metastore/events/PreReadDatabaseEvent.java | 46 -
.../metastore/events/PreReadTableEvent.java | 47 -
.../messaging/AddForeignKeyMessage.java | 36 -
.../messaging/AddNotNullConstraintMessage.java | 36 -
.../messaging/AddPartitionMessage.java | 68 --
.../messaging/AddPrimaryKeyMessage.java | 35 -
.../messaging/AddUniqueConstraintMessage.java | 36 -
.../metastore/messaging/AlterIndexMessage.java | 33 -
.../messaging/AlterPartitionMessage.java | 69 --
.../metastore/messaging/AlterTableMessage.java | 58 --
.../messaging/CreateDatabaseMessage.java | 28 -
.../messaging/CreateFunctionMessage.java | 46 -
.../metastore/messaging/CreateIndexMessage.java | 46 -
.../metastore/messaging/CreateTableMessage.java | 53 --
.../messaging/DropConstraintMessage.java | 29 -
.../messaging/DropDatabaseMessage.java | 27 -
.../messaging/DropFunctionMessage.java | 38 -
.../metastore/messaging/DropIndexMessage.java | 46 -
.../messaging/DropPartitionMessage.java | 49 -
.../metastore/messaging/DropTableMessage.java | 46 -
.../hive/metastore/messaging/EventMessage.java | 111 ---
.../hive/metastore/messaging/InsertMessage.java | 75 --
.../messaging/MessageDeserializer.java | 179 ----
.../metastore/messaging/MessageFactory.java | 291 ------
.../metastore/messaging/PartitionFiles.java | 53 --
.../json/JSONAddForeignKeyMessage.java | 101 ---
.../json/JSONAddNotNullConstraintMessage.java | 96 --
.../messaging/json/JSONAddPartitionMessage.java | 171 ----
.../json/JSONAddPrimaryKeyMessage.java | 101 ---
.../json/JSONAddUniqueConstraintMessage.java | 98 --
.../messaging/json/JSONAlterIndexMessage.java | 98 --
.../json/JSONAlterPartitionMessage.java | 150 ----
.../messaging/json/JSONAlterTableMessage.java | 123 ---
.../json/JSONCreateDatabaseMessage.java | 71 --
.../json/JSONCreateFunctionMessage.java | 86 --
.../messaging/json/JSONCreateIndexMessage.java | 87 --
.../messaging/json/JSONCreateTableMessage.java | 130 ---
.../json/JSONDropConstraintMessage.java | 90 --
.../messaging/json/JSONDropDatabaseMessage.java | 71 --
.../messaging/json/JSONDropFunctionMessage.java | 79 --
.../messaging/json/JSONDropIndexMessage.java | 91 --
.../json/JSONDropPartitionMessage.java | 130 ---
.../messaging/json/JSONDropTableMessage.java | 116 ---
.../messaging/json/JSONInsertMessage.java | 144 ---
.../messaging/json/JSONMessageDeserializer.java | 243 -----
.../messaging/json/JSONMessageFactory.java | 354 --------
.../hive/metastore/TestHiveAlterHandler.java | 108 ---
.../hive/metastore/TestMetaStoreUtils.java | 43 -
.../TestRetriesInRetryingHMSHandler.java | 109 ---
.../json/TestJSONMessageDeserializer.java | 106 ---
.../hive/ql/parse/MetaDataExportListener.java | 16 +-
.../HadoopDefaultMetastoreAuthenticator.java | 4 +-
.../HiveMetastoreAuthenticationProvider.java | 4 +-
...faultHiveMetastoreAuthorizationProvider.java | 4 +-
.../HiveAuthorizationProviderBase.java | 6 +-
.../HiveMetastoreAuthorizationProvider.java | 4 +-
.../MetaStoreAuthzAPIAuthorizerEmbedOnly.java | 4 +-
.../StorageBasedAuthorizationProvider.java | 4 +-
.../ql/txn/AcidCompactionHistoryService.java | 82 --
.../hive/ql/txn/AcidHouseKeeperService.java | 82 --
.../hadoop/hive/ql/txn/AcidWriteSetService.java | 78 --
.../txn/compactor/HouseKeeperServiceBase.java | 96 --
.../apache/hadoop/hive/ql/TestTxnCommands.java | 16 +-
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 30 +-
.../hive/ql/lockmgr/TestDbTxnManager.java | 25 +-
.../hive/ql/lockmgr/TestDbTxnManager2.java | 12 +-
.../org/apache/hadoop/hive/io/HdfsUtils.java | 11 -
standalone-metastore/pom.xml | 6 +
.../hive/metastore/api/ClientCapability.java | 8 +-
.../hive/metastore/AcidEventListener.java | 95 ++
.../hadoop/hive/metastore/AlterHandler.java | 195 ++++
.../hive/metastore/FileMetadataManager.java | 124 +++
.../hive/metastore/HMSMetricsListener.java | 90 ++
.../hadoop/hive/metastore/HiveAlterHandler.java | 897 ++++++++++++++++++
.../hadoop/hive/metastore/IHMSHandler.java | 59 ++
.../hive/metastore/MetaStoreEventListener.java | 227 +++++
.../MetaStoreEventListenerConstants.java | 41 +
.../hive/metastore/MetaStoreInitContext.java | 27 +
.../hive/metastore/MetaStoreInitListener.java | 49 +
.../metastore/MetaStoreListenerNotifier.java | 266 ++++++
.../metastore/MetaStorePreEventListener.java | 57 ++
.../hive/metastore/RetryingHMSHandler.java | 232 +++++
.../metastore/SessionPropertiesListener.java | 46 +
.../TransactionalMetaStoreEventListener.java | 39 +
.../TransactionalValidationListener.java | 360 ++++++++
.../hive/metastore/conf/MetastoreConf.java | 16 +
.../metastore/events/AddForeignKeyEvent.java | 41 +
.../hive/metastore/events/AddIndexEvent.java | 40 +
.../events/AddNotNullConstraintEvent.java | 42 +
.../metastore/events/AddPartitionEvent.java | 84 ++
.../metastore/events/AddPrimaryKeyEvent.java | 42 +
.../events/AddUniqueConstraintEvent.java | 42 +
.../hive/metastore/events/AlterIndexEvent.java | 46 +
.../metastore/events/AlterPartitionEvent.java | 75 ++
.../hive/metastore/events/AlterTableEvent.java | 63 ++
.../metastore/events/ConfigChangeEvent.java | 52 ++
.../metastore/events/CreateDatabaseEvent.java | 43 +
.../metastore/events/CreateFunctionEvent.java | 43 +
.../hive/metastore/events/CreateTableEvent.java | 43 +
.../metastore/events/DropConstraintEvent.java | 51 ++
.../metastore/events/DropDatabaseEvent.java | 43 +
.../metastore/events/DropFunctionEvent.java | 43 +
.../hive/metastore/events/DropIndexEvent.java | 40 +
.../metastore/events/DropPartitionEvent.java | 70 ++
.../hive/metastore/events/DropTableEvent.java | 54 ++
.../hive/metastore/events/EventCleanerTask.java | 52 ++
.../hive/metastore/events/InsertEvent.java | 129 +++
.../hive/metastore/events/ListenerEvent.java | 178 ++++
.../events/LoadPartitionDoneEvent.java | 57 ++
.../hive/metastore/events/PreAddIndexEvent.java | 41 +
.../metastore/events/PreAddPartitionEvent.java | 79 ++
.../metastore/events/PreAlterDatabaseEvent.java | 47 +
.../metastore/events/PreAlterIndexEvent.java | 47 +
.../events/PreAlterPartitionEvent.java | 65 ++
.../metastore/events/PreAlterTableEvent.java | 53 ++
.../events/PreAuthorizationCallEvent.java | 33 +
.../events/PreCreateDatabaseEvent.java | 43 +
.../metastore/events/PreCreateTableEvent.java | 43 +
.../metastore/events/PreDropDatabaseEvent.java | 43 +
.../metastore/events/PreDropIndexEvent.java | 40 +
.../metastore/events/PreDropPartitionEvent.java | 67 ++
.../metastore/events/PreDropTableEvent.java | 55 ++
.../hive/metastore/events/PreEventContext.java | 73 ++
.../events/PreLoadPartitionDoneEvent.java | 58 ++
.../metastore/events/PreReadDatabaseEvent.java | 46 +
.../metastore/events/PreReadTableEvent.java | 47 +
.../messaging/AddForeignKeyMessage.java | 36 +
.../messaging/AddNotNullConstraintMessage.java | 36 +
.../messaging/AddPartitionMessage.java | 68 ++
.../messaging/AddPrimaryKeyMessage.java | 35 +
.../messaging/AddUniqueConstraintMessage.java | 36 +
.../metastore/messaging/AlterIndexMessage.java | 33 +
.../messaging/AlterPartitionMessage.java | 69 ++
.../metastore/messaging/AlterTableMessage.java | 58 ++
.../messaging/CreateDatabaseMessage.java | 28 +
.../messaging/CreateFunctionMessage.java | 46 +
.../metastore/messaging/CreateIndexMessage.java | 46 +
.../metastore/messaging/CreateTableMessage.java | 53 ++
.../messaging/DropConstraintMessage.java | 29 +
.../messaging/DropDatabaseMessage.java | 27 +
.../messaging/DropFunctionMessage.java | 38 +
.../metastore/messaging/DropIndexMessage.java | 46 +
.../messaging/DropPartitionMessage.java | 49 +
.../metastore/messaging/DropTableMessage.java | 46 +
.../hive/metastore/messaging/EventMessage.java | 111 +++
.../hive/metastore/messaging/InsertMessage.java | 75 ++
.../messaging/MessageDeserializer.java | 179 ++++
.../metastore/messaging/MessageFactory.java | 289 ++++++
.../metastore/messaging/PartitionFiles.java | 53 ++
.../json/JSONAddForeignKeyMessage.java | 101 +++
.../json/JSONAddNotNullConstraintMessage.java | 96 ++
.../messaging/json/JSONAddPartitionMessage.java | 171 ++++
.../json/JSONAddPrimaryKeyMessage.java | 101 +++
.../json/JSONAddUniqueConstraintMessage.java | 98 ++
.../messaging/json/JSONAlterIndexMessage.java | 98 ++
.../json/JSONAlterPartitionMessage.java | 148 +++
.../messaging/json/JSONAlterTableMessage.java | 123 +++
.../json/JSONCreateDatabaseMessage.java | 71 ++
.../json/JSONCreateFunctionMessage.java | 86 ++
.../messaging/json/JSONCreateIndexMessage.java | 87 ++
.../messaging/json/JSONCreateTableMessage.java | 130 +++
.../json/JSONDropConstraintMessage.java | 90 ++
.../messaging/json/JSONDropDatabaseMessage.java | 71 ++
.../messaging/json/JSONDropFunctionMessage.java | 78 ++
.../messaging/json/JSONDropIndexMessage.java | 90 ++
.../json/JSONDropPartitionMessage.java | 130 +++
.../messaging/json/JSONDropTableMessage.java | 116 +++
.../messaging/json/JSONInsertMessage.java | 144 +++
.../messaging/json/JSONMessageDeserializer.java | 243 +++++
.../messaging/json/JSONMessageFactory.java | 354 ++++++++
.../txn/AcidCompactionHistoryService.java | 62 ++
.../metastore/txn/AcidHouseKeeperService.java | 63 ++
.../hive/metastore/txn/AcidWriteSetService.java | 61 ++
.../hadoop/hive/metastore/txn/TxnDbUtil.java | 6 +
.../hadoop/hive/metastore/utils/FileUtils.java | 16 +
.../hadoop/hive/metastore/utils/HdfsUtils.java | 21 +
.../hadoop/hive/metastore/utils/JavaUtils.java | 14 +
.../hive/metastore/utils/MetaStoreUtils.java | 390 +++++++-
.../hive/metastore/TestHiveAlterHandler.java | 108 +++
.../TestRetriesInRetryingHMSHandler.java | 108 +++
.../json/TestJSONMessageDeserializer.java | 106 +++
.../metastore/utils/TestMetaStoreUtils.java | 84 ++
244 files changed, 10351 insertions(+), 10260 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c5a9673a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java
----------------------------------------------------------------------
diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java
index f7c2f44..66349c5 100644
--- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java
+++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java
@@ -34,12 +34,9 @@ import javax.naming.InitialContext;
import javax.naming.NamingException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.metastore.IHMSHandler;
import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
@@ -54,6 +51,7 @@ import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.messaging.HCatEventMessage;
import org.apache.hive.hcatalog.messaging.MessageFactory;
+import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -182,7 +180,7 @@ public class NotificationListener extends MetaStoreEventListener {
// by listening on a topic named "HCAT" and message selector string
// as "HCAT_EVENT = HCAT_ADD_DATABASE"
if (dbEvent.getStatus()) {
- String topicName = getTopicPrefix(dbEvent.getHandler().getHiveConf());
+ String topicName = getTopicPrefix(dbEvent.getIHMSHandler().getConf());
send(messageFactory.buildCreateDatabaseMessage(dbEvent.getDatabase()), topicName);
}
}
@@ -193,7 +191,7 @@ public class NotificationListener extends MetaStoreEventListener {
// by listening on a topic named "HCAT" and message selector string
// as "HCAT_EVENT = HCAT_DROP_DATABASE"
if (dbEvent.getStatus()) {
- String topicName = getTopicPrefix(dbEvent.getHandler().getHiveConf());
+ String topicName = getTopicPrefix(dbEvent.getIHMSHandler().getConf());
send(messageFactory.buildDropDatabaseMessage(dbEvent.getDatabase()), topicName);
}
}
@@ -205,8 +203,8 @@ public class NotificationListener extends MetaStoreEventListener {
// as "HCAT_EVENT = HCAT_ADD_TABLE"
if (tableEvent.getStatus()) {
Table tbl = tableEvent.getTable();
- HMSHandler handler = tableEvent.getHandler();
- HiveConf conf = handler.getHiveConf();
+ IHMSHandler handler = tableEvent.getIHMSHandler();
+ Configuration conf = handler.getConf();
Table newTbl;
try {
newTbl = handler.get_table_core(tbl.getDbName(), tbl.getTableName())
@@ -216,11 +214,7 @@ public class NotificationListener extends MetaStoreEventListener {
getTopicPrefix(conf) + "." + newTbl.getDbName().toLowerCase() + "."
+ newTbl.getTableName().toLowerCase());
handler.alter_table(newTbl.getDbName(), newTbl.getTableName(), newTbl);
- } catch (InvalidOperationException e) {
- MetaException me = new MetaException(e.toString());
- me.initCause(e);
- throw me;
- } catch (NoSuchObjectException e) {
+ } catch (TException e) {
MetaException me = new MetaException(e.toString());
me.initCause(e);
throw me;
@@ -258,7 +252,7 @@ public class NotificationListener extends MetaStoreEventListener {
}
// I think this is wrong, the alter table statement should come on the table topic not the
// DB topic - Alan.
- String topicName = getTopicPrefix(tableEvent.getHandler().getHiveConf()) + "." +
+ String topicName = getTopicPrefix(tableEvent.getIHMSHandler().getConf()) + "." +
after.getDbName().toLowerCase();
send(messageFactory.buildAlterTableMessage(before, after), topicName);
}
@@ -288,7 +282,7 @@ public class NotificationListener extends MetaStoreEventListener {
Table table = tableEvent.getTable();
// I think this is wrong, the drop table statement should come on the table topic not the
// DB topic - Alan.
- String topicName = getTopicPrefix(tableEvent.getHandler().getHiveConf()) + "." + table.getDbName().toLowerCase();
+ String topicName = getTopicPrefix(tableEvent.getIHMSHandler().getConf()) + "." + table.getDbName().toLowerCase();
send(messageFactory.buildDropTableMessage(table), topicName);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c5a9673a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
index 3b5066f..49aad39 100644
--- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
+++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
import org.apache.hadoop.hive.metastore.api.TxnInfo;
import org.apache.hadoop.hive.metastore.api.TxnState;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService;
import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
@@ -77,7 +78,6 @@ import org.apache.hadoop.hive.ql.io.orc.Reader;
import org.apache.hadoop.hive.ql.io.orc.RecordReader;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService;
import org.apache.hadoop.hive.ql.txn.compactor.Worker;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
@@ -740,11 +740,8 @@ public class TestStreaming {
//ensure txn timesout
conf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 1, TimeUnit.MILLISECONDS);
AcidHouseKeeperService houseKeeperService = new AcidHouseKeeperService();
- houseKeeperService.start(conf);
- while(houseKeeperService.getIsAliveCounter() <= Integer.MIN_VALUE) {
- Thread.sleep(100);//make sure it has run at least once
- }
- houseKeeperService.stop();
+ houseKeeperService.setConf(conf);
+ houseKeeperService.run();
try {
//should fail because the TransactionBatch timed out
txnBatch.commit();
@@ -757,12 +754,7 @@ public class TestStreaming {
txnBatch.beginNextTransaction();
txnBatch.commit();
txnBatch.beginNextTransaction();
- int lastCount = houseKeeperService.getIsAliveCounter();
- houseKeeperService.start(conf);
- while(houseKeeperService.getIsAliveCounter() <= lastCount) {
- Thread.sleep(100);//make sure it has run at least once
- }
- houseKeeperService.stop();
+ houseKeeperService.run();
try {
//should fail because the TransactionBatch timed out
txnBatch.commit();
http://git-wip-us.apache.org/repos/asf/hive/blob/c5a9673a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
index 42e476a..4df2f86 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
@@ -509,13 +509,13 @@ public class TestMetaStoreEventListener extends TestCase {
closingClient.setMetaConf(metaConfKey, "[test pattern modified]");
ConfigChangeEvent event = (ConfigChangeEvent) DummyListener.getLastEvent();
int beforeCloseNotificationEventCounts = DummyListener.notifyList.size();
- HiveMetaStore.HMSHandler beforeHandler = event.getHandler();
+ IHMSHandler beforeHandler = event.getIHMSHandler();
closingClient.close();
Thread.sleep(5 * 1000);
event = (ConfigChangeEvent) DummyListener.getLastEvent();
int afterCloseNotificationEventCounts = DummyListener.notifyList.size();
- HiveMetaStore.HMSHandler afterHandler = event.getHandler();
+ IHMSHandler afterHandler = event.getIHMSHandler();
// Meta-conf cleanup should trigger an event to listener
assertNotSame(beforeCloseNotificationEventCounts, afterCloseNotificationEventCounts);
// Both the handlers should be same
http://git-wip-us.apache.org/repos/asf/hive/blob/c5a9673a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java
deleted file mode 100644
index c4dd97e..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import junit.framework.TestCase;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class TestMetaStoreUtils extends TestCase {
-
- public void testTrimMapNullsXform() throws Exception {
- Map<String,String> m = new HashMap<String,String>();
- m.put("akey","aval");
- m.put("blank","");
- m.put("null",null);
-
- Map<String,String> xformed = MetaStoreUtils.trimMapNulls(m,true);
- assertEquals(3,xformed.size());
- assert(xformed.containsKey("akey"));
- assert(xformed.containsKey("blank"));
- assert(xformed.containsKey("null"));
- assertEquals("aval",xformed.get("akey"));
- assertEquals("",xformed.get("blank"));
- assertEquals("",xformed.get("null"));
- }
-
- public void testTrimMapNullsPrune() throws Exception {
- Map<String,String> m = new HashMap<String,String>();
- m.put("akey","aval");
- m.put("blank","");
- m.put("null",null);
-
- Map<String,String> pruned = MetaStoreUtils.trimMapNulls(m,false);
- assertEquals(2,pruned.size());
- assert(pruned.containsKey("akey"));
- assert(pruned.containsKey("blank"));
- assert(!pruned.containsKey("null"));
- assertEquals("aval",pruned.get("akey"));
- assertEquals("",pruned.get("blank"));
- assert(!pruned.containsValue(null));
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/c5a9673a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java
index 93637d4..6662fcd 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.metastore.IHMSHandler;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.ql.metadata.AuthorizationException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -201,7 +201,7 @@ public class DummyHiveMetastoreAuthorizationProvider implements HiveMetastoreAut
}
@Override
- public void setMetaStoreHandler(HMSHandler handler) {
+ public void setMetaStoreHandler(IHMSHandler handler) {
debugLog("DHMAP.setMetaStoreHandler");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c5a9673a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java
index 322834e..d4cf026 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hive.ql.security;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.metastore.IHMSHandler;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -91,7 +91,7 @@ public class InjectableDummyAuthenticator implements HiveMetastoreAuthentication
}
@Override
- public void setMetaStoreHandler(HMSHandler handler) {
+ public void setMetaStoreHandler(IHMSHandler handler) {
hmap.setMetaStoreHandler(handler);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c5a9673a/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java b/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
deleted file mode 100644
index 71ad916..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.HiveObjectType;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
-import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
-import org.apache.hadoop.hive.metastore.events.DropTableEvent;
-import org.apache.hadoop.hive.metastore.txn.TxnStore;
-import org.apache.hadoop.hive.metastore.txn.TxnUtils;
-
-
-/**
- * It handles cleanup of dropped partition/table/database in ACID related metastore tables
- */
-public class AcidEventListener extends MetaStoreEventListener {
-
- private TxnStore txnHandler;
- private HiveConf hiveConf;
-
- public AcidEventListener(Configuration configuration) {
- super(configuration);
- hiveConf = (HiveConf) configuration;
- }
-
- @Override
- public void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException {
- // We can loop thru all the tables to check if they are ACID first and then perform cleanup,
- // but it's more efficient to unconditionally perform cleanup for the database, especially
- // when there are a lot of tables
- txnHandler = getTxnHandler();
- txnHandler.cleanupRecords(HiveObjectType.DATABASE, dbEvent.getDatabase(), null, null);
- }
-
- @Override
- public void onDropTable(DropTableEvent tableEvent) throws MetaException {
- if (TxnUtils.isAcidTable(tableEvent.getTable())) {
- txnHandler = getTxnHandler();
- txnHandler.cleanupRecords(HiveObjectType.TABLE, null, tableEvent.getTable(), null);
- }
- }
-
- @Override
- public void onDropPartition(DropPartitionEvent partitionEvent) throws MetaException {
- if (TxnUtils.isAcidTable(partitionEvent.getTable())) {
- txnHandler = getTxnHandler();
- txnHandler.cleanupRecords(HiveObjectType.PARTITION, null, partitionEvent.getTable(),
- partitionEvent.getPartitionIterator());
- }
- }
-
- private TxnStore getTxnHandler() {
- boolean hackOn = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_IN_TEST) ||
- HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_IN_TEZ_TEST);
- String origTxnMgr = null;
- boolean origConcurrency = false;
-
- // Since TxnUtils.getTxnStore calls TxnHandler.setConf -> checkQFileTestHack -> TxnDbUtil.setConfValues,
- // which may change the values of below two entries, we need to avoid pulluting the original values
- if (hackOn) {
- origTxnMgr = hiveConf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER);
- origConcurrency = hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY);
- }
-
- txnHandler = TxnUtils.getTxnStore(hiveConf);
-
- // Set them back
- if (hackOn) {
- hiveConf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, origTxnMgr);
- hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, origConcurrency);
- }
-
- return txnHandler;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/c5a9673a/metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java
deleted file mode 100644
index a3d322f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import java.util.List;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
-import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
-
-/**
- * Interface for Alter Table and Alter Partition code
- */
-public interface AlterHandler extends Configurable {
-
- /**
- * @deprecated As of release 2.2.0. Replaced by {@link #alterTable(RawStore, Warehouse, String,
- * String, Table, EnvironmentContext, HMSHandler)}
- *
- * handles alter table, the changes could be cascaded to partitions if applicable
- *
- * @param msdb
- * object to get metadata
- * @param wh
- * Hive Warehouse where table data is stored
- * @param dbname
- * database of the table being altered
- * @param name
- * original name of the table being altered. same as
- * <i>newTable.tableName</i> if alter op is not a rename.
- * @param newTable
- * new table object
- * @throws InvalidOperationException
- * thrown if the newTable object is invalid
- * @throws MetaException
- * thrown if there is any other error
- */
- @Deprecated
- void alterTable(RawStore msdb, Warehouse wh, String dbname,
- String name, Table newTable, EnvironmentContext envContext)
- throws InvalidOperationException, MetaException;
-
- /**
- * handles alter table, the changes could be cascaded to partitions if applicable
- *
- * @param msdb
- * object to get metadata
- * @param wh
- * Hive Warehouse where table data is stored
- * @param dbname
- * database of the table being altered
- * @param name
- * original name of the table being altered. same as
- * <i>newTable.tableName</i> if alter op is not a rename.
- * @param newTable
- * new table object
- * @param handler
- * HMSHandle object (required to log event notification)
- * @throws InvalidOperationException
- * thrown if the newTable object is invalid
- * @throws MetaException
- * thrown if there is any other error
- */
- void alterTable(RawStore msdb, Warehouse wh, String dbname,
- String name, Table newTable, EnvironmentContext envContext,
- HMSHandler handler) throws InvalidOperationException, MetaException;
-
- /**
- * @deprecated As of release 2.2.0. Replaced by {@link #alterPartition(RawStore, Warehouse, String,
- * String, List, Partition, EnvironmentContext, HMSHandler)}
- *
- * handles alter partition
- *
- * @param msdb
- * object to get metadata
- * @param wh
- * @param dbname
- * database of the partition being altered
- * @param name
- * table of the partition being altered
- * @param part_vals
- * original values of the partition being altered
- * @param new_part
- * new partition object
- * @return the altered partition
- * @throws InvalidOperationException
- * @throws InvalidObjectException
- * @throws AlreadyExistsException
- * @throws MetaException
- */
- @Deprecated
- Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
- final String name, final List<String> part_vals, final Partition new_part,
- EnvironmentContext environmentContext)
- throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException;
-
- /**
- * handles alter partition
- *
- * @param msdb
- * object to get metadata
- * @param wh
- * @param dbname
- * database of the partition being altered
- * @param name
- * table of the partition being altered
- * @param part_vals
- * original values of the partition being altered
- * @param new_part
- * new partition object
- * @param handler
- * HMSHandle object (required to log event notification)
- * @return the altered partition
- * @throws InvalidOperationException
- * @throws InvalidObjectException
- * @throws AlreadyExistsException
- * @throws MetaException
- */
- Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
- final String name, final List<String> part_vals, final Partition new_part, EnvironmentContext environmentContext,
- HMSHandler handler)
- throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException;
-
- /**
- * @deprecated As of release 2.2.0. Replaced by {@link #alterPartitions(RawStore, Warehouse, String,
- * String, List, EnvironmentContext, HMSHandler)}
- *
- * handles alter partitions
- *
- * @param msdb
- * object to get metadata
- * @param wh
- * @param dbname
- * database of the partition being altered
- * @param name
- * table of the partition being altered
- * @param new_parts
- * new partition list
- * @return the altered partition list
- * @throws InvalidOperationException
- * @throws InvalidObjectException
- * @throws AlreadyExistsException
- * @throws MetaException
- */
- @Deprecated
- List<Partition> alterPartitions(final RawStore msdb, Warehouse wh,
- final String dbname, final String name, final List<Partition> new_parts,
- EnvironmentContext environmentContext)
- throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException;
-
- /**
- * handles alter partitions
- *
- * @param msdb
- * object to get metadata
- * @param wh
- * @param dbname
- * database of the partition being altered
- * @param name
- * table of the partition being altered
- * @param new_parts
- * new partition list
- * @param handler
- * HMSHandle object (required to log event notification)
- * @return the altered partition list
- * @throws InvalidOperationException
- * @throws InvalidObjectException
- * @throws AlreadyExistsException
- * @throws MetaException
- */
- List<Partition> alterPartitions(final RawStore msdb, Warehouse wh,
- final String dbname, final String name, final List<Partition> new_parts,
- EnvironmentContext environmentContext,HMSHandler handler)
- throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/c5a9673a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java b/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java
deleted file mode 100644
index 67aa144..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-
-import org.apache.hadoop.fs.LocatedFileStatus;
-
-import org.apache.hadoop.fs.RemoteIterator;
-
-import java.util.LinkedList;
-
-import java.util.Queue;
-
-import java.util.ArrayList;
-
-import java.util.List;
-
-import com.google.common.collect.Lists;
-
-import java.io.IOException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.io.HdfsUtils;
-import org.apache.hadoop.hive.metastore.HiveMetaStore.ThreadLocalRawStore;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.shims.HadoopShims;
-import org.apache.hadoop.hive.shims.ShimLoader;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-public class FileMetadataManager {
- private static final Log LOG = LogFactory.getLog(FileMetadataManager.class);
- private static final HadoopShims SHIMS = ShimLoader.getHadoopShims();
-
- private final ThreadLocalRawStore tlms;
- private final ExecutorService threadPool;
- private final HiveConf conf;
-
- private final class CacheUpdateRequest implements Callable<Void> {
- FileMetadataExprType type;
- String location;
-
- public CacheUpdateRequest(FileMetadataExprType type, String location) {
- this.type = type;
- this.location = location;
- }
-
- @Override
- public Void call() throws Exception {
- try {
- cacheMetadata(type, location);
- } catch (InterruptedException ex) {
- Thread.currentThread().interrupt();
- } catch (Exception ex) {
- // Nobody can see this exception on the threadpool; just log it.
- LOG.error("Failed to cache file metadata in background for " + type + ", " + location, ex);
- }
- return null;
- }
- }
-
- public FileMetadataManager(ThreadLocalRawStore tlms, HiveConf conf) {
- this.tlms = tlms;
- this.conf = conf;
- int numThreads = HiveConf.getIntVar(conf, ConfVars.METASTORE_HBASE_FILE_METADATA_THREADS);
- this.threadPool = Executors.newFixedThreadPool(numThreads,
- new ThreadFactoryBuilder().setNameFormat("File-Metadata-%d").setDaemon(true).build());
- }
-
- public void queueCacheMetadata(String location, FileMetadataExprType type) {
- threadPool.submit(new CacheUpdateRequest(type, location));
- }
-
- private void cacheMetadata(FileMetadataExprType type, String location)
- throws MetaException, IOException, InterruptedException {
- Path path = new Path(location);
- FileSystem fs = path.getFileSystem(conf);
- List<Path> files;
- if (!fs.isDirectory(path)) {
- files = Lists.newArrayList(path);
- } else {
- files = new ArrayList<>();
- RemoteIterator<LocatedFileStatus> iter = fs.listFiles(path, true);
- while (iter.hasNext()) {
- // TODO: use fileId right from the list after HDFS-7878; or get dfs client and do it
- LocatedFileStatus lfs = iter.next();
- if (lfs.isDirectory()) continue;
- files.add(lfs.getPath());
- }
- }
- for (Path file : files) {
- long fileId;
- // TODO: use the other HdfsUtils here
- if (!(fs instanceof DistributedFileSystem)) return;
- try {
- fileId = SHIMS.getFileId((DistributedFileSystem)fs,
- Path.getPathWithoutSchemeAndAuthority(file).toString());
- } catch (UnsupportedOperationException ex) {
- LOG.error("Cannot cache file metadata for " + location + "; "
- + fs.getClass().getCanonicalName() + " does not support fileId");
- return;
- }
- LOG.info("Caching file metadata for " + file + " (file ID " + fileId + ")");
- file = HdfsUtils.getFileIdPath(fs, file, fileId);
- tlms.getMS().getFileMetadataHandler(type).cacheFileMetadata(fileId, fs, file);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/c5a9673a/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
deleted file mode 100644
index a2ad06f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.MetricRegistry;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
-import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
-import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
-import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
-import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
-import org.apache.hadoop.hive.metastore.events.DropTableEvent;
-import org.apache.hadoop.hive.metastore.metrics.Metrics;
-import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Report metrics of metadata added, deleted by this Hive Metastore.
- */
-public class HMSMetricsListener extends MetaStoreEventListener {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(HMSMetricsListener.class);
-
- private Counter createdDatabases, deletedDatabases, createdTables, deletedTables, createdParts,
- deletedParts;
-
- public HMSMetricsListener(Configuration config) {
- super(config);
- createdDatabases = Metrics.getOrCreateCounter(MetricsConstants.CREATE_TOTAL_DATABASES);
- deletedDatabases = Metrics.getOrCreateCounter(MetricsConstants.DELETE_TOTAL_DATABASES);
- createdTables = Metrics.getOrCreateCounter(MetricsConstants.CREATE_TOTAL_TABLES);
- deletedTables = Metrics.getOrCreateCounter(MetricsConstants.DELETE_TOTAL_TABLES);
- createdParts = Metrics.getOrCreateCounter(MetricsConstants.CREATE_TOTAL_PARTITIONS);
- deletedParts = Metrics.getOrCreateCounter(MetricsConstants.DELETE_TOTAL_PARTITIONS);
- }
-
- @Override
- public void onCreateDatabase(CreateDatabaseEvent dbEvent) throws MetaException {
- HiveMetaStore.HMSHandler.databaseCount.incrementAndGet();
- createdDatabases.inc();
- }
-
- @Override
- public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
- HiveMetaStore.HMSHandler.databaseCount.decrementAndGet();
- deletedDatabases.inc();
- }
-
- @Override
- public void onCreateTable(CreateTableEvent tableEvent) throws MetaException {
- HiveMetaStore.HMSHandler.tableCount.incrementAndGet();
- createdTables.inc();
- }
-
- @Override
- public void onDropTable(DropTableEvent tableEvent) throws MetaException {
- HiveMetaStore.HMSHandler.tableCount.decrementAndGet();
- deletedTables.inc();
- }
-
- @Override
- public void onDropPartition(DropPartitionEvent partitionEvent) throws MetaException {
- HiveMetaStore.HMSHandler.partCount.decrementAndGet();
- deletedParts.inc();
- }
-
- @Override
- public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException {
- HiveMetaStore.HMSHandler.partCount.incrementAndGet();
- createdParts.inc();
- }
-}