You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2018/08/10 04:25:37 UTC
hive git commit: HIVE-20316: Skip external table file listing for
create table event (Sankar Hariappan, reviewed by Mahesh Kumar Behera,
Anishek Agarwal)
Repository: hive
Updated Branches:
refs/heads/master 3dc736fcf -> 6286bbc35
HIVE-20316: Skip external table file listing for create table event (Sankar Hariappan, reviewed by Mahesh Kumar Behera, Anishek Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6286bbc3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6286bbc3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6286bbc3
Branch: refs/heads/master
Commit: 6286bbc35eebca76ab4def729711204bd989e0e2
Parents: 3dc736f
Author: Sankar Hariappan <sa...@apache.org>
Authored: Fri Aug 10 09:55:05 2018 +0530
Committer: Sankar Hariappan <sa...@apache.org>
Committed: Fri Aug 10 09:55:05 2018 +0530
----------------------------------------------------------------------
.../hive/hcatalog/listener/DbNotificationListener.java | 12 ++++++++----
.../parse/TestReplicationScenariosAcrossInstances.java | 5 ++++-
.../messaging/json/JSONAddPartitionMessage.java | 3 ++-
.../messaging/json/JSONCreateTableMessage.java | 2 +-
4 files changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6286bbc3/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
----------------------------------------------------------------------
diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
index 717cc8a..2ab59d7 100644
--- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
+++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
@@ -84,6 +84,7 @@ import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage;
import org.apache.hadoop.hive.metastore.messaging.PartitionFiles;
import org.apache.hadoop.hive.metastore.tools.SQLGenerator;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -155,9 +156,11 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener
@Override
public void onCreateTable(CreateTableEvent tableEvent) throws MetaException {
Table t = tableEvent.getTable();
+ FileIterator fileIter = MetaStoreUtils.isExternalTable(t)
+ ? null : new FileIterator(t.getSd().getLocation());
NotificationEvent event =
- new NotificationEvent(0, now(), EventType.CREATE_TABLE.toString(), msgFactory
- .buildCreateTableMessage(t, new FileIterator(t.getSd().getLocation())).toString());
+ new NotificationEvent(0, now(), EventType.CREATE_TABLE.toString(),
+ msgFactory.buildCreateTableMessage(t, fileIter).toString());
event.setCatName(t.isSetCatName() ? t.getCatName() : DEFAULT_CATALOG_NAME);
event.setDbName(t.getDbName());
event.setTableName(t.getTableName());
@@ -301,9 +304,10 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener
@Override
public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException {
Table t = partitionEvent.getTable();
+ PartitionFilesIterator fileIter = MetaStoreUtils.isExternalTable(t)
+ ? null : new PartitionFilesIterator(partitionEvent.getPartitionIterator(), t);
String msg = msgFactory
- .buildAddPartitionMessage(t, partitionEvent.getPartitionIterator(),
- new PartitionFilesIterator(partitionEvent.getPartitionIterator(), t)).toString();
+ .buildAddPartitionMessage(t, partitionEvent.getPartitionIterator(), fileIter).toString();
NotificationEvent event =
new NotificationEvent(0, now(), EventType.ADD_PARTITION.toString(), msg);
event.setCatName(t.isSetCatName() ? t.getCatName() : DEFAULT_CATALOG_NAME);
http://git-wip-us.apache.org/repos/asf/hive/blob/6286bbc3/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index b2e1e25..6c45641 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -1387,6 +1387,7 @@ public class TestReplicationScenariosAcrossInstances {
tuple = primary.run("use " + primaryDbName)
.run("create external table t3 (id int)")
.run("insert into table t3 values (10)")
+ .run("create external table t4 as select id from t3")
.dump("repl dump " + primaryDbName + " from " + tuple.lastReplicationId
+ " with ('hive.repl.include.external.tables'='true')");
@@ -1395,6 +1396,8 @@ public class TestReplicationScenariosAcrossInstances {
.run("show tables like 't3'")
.verifyResult("t3")
.run("select id from t3")
- .verifyResult("10");
+ .verifyResult("10")
+ .run("select id from t4")
+ .verifyResult(null); // Returns null as create table event doesn't list files
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6286bbc3/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
index 11921f2..bb2093b 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
@@ -88,7 +88,8 @@ public class JSONAddPartitionMessage extends AddPartitionMessage {
} catch (TException e) {
throw new IllegalArgumentException("Could not serialize: ", e);
}
- this.partitionFiles = Lists.newArrayList(partitionFileIter);
+ this.partitionFiles = (partitionFileIter != null) ? Lists.newArrayList(partitionFileIter)
+ : Lists.newArrayList();
checkValid();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6286bbc3/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
index d0f4b86..b80003b 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
@@ -72,7 +72,7 @@ public class JSONCreateTableMessage extends CreateTableMessage {
} catch (TException e) {
throw new IllegalArgumentException("Could not serialize: ", e);
}
- this.files = Lists.newArrayList(fileIter);
+ this.files = (fileIter != null) ? Lists.newArrayList(fileIter) : Lists.newArrayList();
}
@Override