You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2016/05/19 20:00:34 UTC
[5/5] hive git commit: HIVE-13622 WriteSet tracking optimizations
(Eugene Koifman, reviewed by Alan Gates)
HIVE-13622 WriteSet tracking optimizations (Eugene Koifman, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c0b532fc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c0b532fc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c0b532fc
Branch: refs/heads/branch-1
Commit: c0b532fceec3ae06f06eb94c0075ac7e86187c35
Parents: 6c65832
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Thu May 19 12:50:06 2016 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Thu May 19 12:50:06 2016 -0700
----------------------------------------------------------------------
.../hive/hcatalog/streaming/HiveEndPoint.java | 4 +-
.../hive/metastore/TestHiveMetaStoreTxns.java | 7 +
metastore/if/hive_metastore.thrift | 12 +
.../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 1814 +++++++++---------
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 905 +++++----
.../gen/thrift/gen-cpp/hive_metastore_types.h | 52 +-
.../metastore/api/AddDynamicPartitions.java | 127 +-
.../hive/metastore/api/DataOperationType.java | 57 +
.../hive/metastore/api/LockComponent.java | 234 ++-
.../src/gen/thrift/gen-php/metastore/Types.php | 86 +
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 69 +-
.../gen/thrift/gen-rb/hive_metastore_types.rb | 27 +-
.../hive/metastore/HiveMetaStoreClient.java | 9 +
.../hadoop/hive/metastore/IMetaStoreClient.java | 11 +-
.../hive/metastore/LockComponentBuilder.java | 10 +
.../hive/metastore/LockRequestBuilder.java | 9 +
.../hadoop/hive/metastore/txn/TxnHandler.java | 108 +-
.../metastore/txn/TestCompactionTxnHandler.java | 16 +-
.../hive/metastore/txn/TestTxnHandler.java | 84 +
.../apache/hadoop/hive/ql/exec/MoveTask.java | 3 +-
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 18 +-
.../hadoop/hive/ql/lockmgr/DbTxnManager.java | 34 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 6 +-
.../hive/ql/lockmgr/TestDbTxnManager2.java | 181 +-
.../hive/ql/txn/compactor/TestCleaner.java | 6 +
.../hive/ql/txn/compactor/TestInitiator.java | 20 +
.../gen-py/hive_service/ThriftHive-remote | 7 +
27 files changed, 2473 insertions(+), 1443 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c0b532fc/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index 41d2834..b9c1e23 100644
--- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.LockComponentBuilder;
import org.apache.hadoop.hive.metastore.LockRequestBuilder;
import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.DataOperationType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
import org.apache.hadoop.hive.metastore.api.LockRequest;
@@ -964,7 +965,8 @@ public class HiveEndPoint {
LockComponentBuilder lockCompBuilder = new LockComponentBuilder()
.setDbName(hiveEndPoint.database)
.setTableName(hiveEndPoint.table)
- .setShared();
+ .setShared()
+ .setOperationType(DataOperationType.INSERT);
if (partNameForLock!=null && !partNameForLock.isEmpty() ) {
lockCompBuilder.setPartitionName(partNameForLock);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c0b532fc/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
index d5ecf98..f0646f6 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
@@ -21,6 +21,7 @@ import junit.framework.Assert;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidReadTxnList;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.DataOperationType;
import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
import org.apache.hadoop.hive.metastore.api.LockResponse;
import org.apache.hadoop.hive.metastore.api.LockState;
@@ -151,14 +152,17 @@ public class TestHiveMetaStoreTxns {
.setTableName("mytable")
.setPartitionName("mypartition")
.setExclusive()
+ .setOperationType(DataOperationType.NO_TXN)
.build());
rqstBuilder.addLockComponent(new LockComponentBuilder()
.setDbName("mydb")
.setTableName("yourtable")
.setSemiShared()
+ .setOperationType(DataOperationType.NO_TXN)
.build());
rqstBuilder.addLockComponent(new LockComponentBuilder()
.setDbName("yourdb")
+ .setOperationType(DataOperationType.NO_TXN)
.setShared()
.build());
rqstBuilder.setUser("fred");
@@ -187,15 +191,18 @@ public class TestHiveMetaStoreTxns {
.setTableName("mytable")
.setPartitionName("mypartition")
.setSemiShared()
+ .setOperationType(DataOperationType.UPDATE)
.build())
.addLockComponent(new LockComponentBuilder()
.setDbName("mydb")
.setTableName("yourtable")
.setSemiShared()
+ .setOperationType(DataOperationType.UPDATE)
.build())
.addLockComponent(new LockComponentBuilder()
.setDbName("yourdb")
.setShared()
+ .setOperationType(DataOperationType.SELECT)
.build())
.setUser("fred");
http://git-wip-us.apache.org/repos/asf/hive/blob/c0b532fc/metastore/if/hive_metastore.thrift
----------------------------------------------------------------------
diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift
index 4b5d207..f3d1d42 100755
--- a/metastore/if/hive_metastore.thrift
+++ b/metastore/if/hive_metastore.thrift
@@ -106,6 +106,15 @@ enum GrantRevokeType {
REVOKE = 2,
}
+enum DataOperationType {
+ SELECT = 1,
+ INSERT = 2
+ UPDATE = 3,
+ DELETE = 4,
+ UNSET = 5,//this is the default to distinguish from NULL from old clients
+ NO_TXN = 6,//drop table, insert overwrite, etc - something non-transactional
+}
+
// Types of events the client can request that the metastore fire. For now just support DML operations, as the metastore knows
// about DDL operations and there's no reason for the client to request such an event.
enum EventRequestType {
@@ -587,6 +596,8 @@ struct LockComponent {
3: required string dbname,
4: optional string tablename,
5: optional string partitionname,
+ 6: optional DataOperationType operationType = DataOperationType.UNSET,
+ 7: optional bool isAcid = false
}
struct LockRequest {
@@ -692,6 +703,7 @@ struct AddDynamicPartitions {
2: required string dbname,
3: required string tablename,
4: required list<string> partitionnames,
+ 5: optional DataOperationType operationType = DataOperationType.UNSET
}
struct NotificationEventRequest {