You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2017/06/08 23:40:31 UTC
[5/5] hive git commit: HIVE-16644 : Hook Change Manager to Insert
Overwrite (Sankar Hariappan via Thejas Nair)
HIVE-16644 : Hook Change Manager to Insert Overwrite (Sankar Hariappan via Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a1869692
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a1869692
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a1869692
Branch: refs/heads/master
Commit: a1869692a29b478635f4f47a81637bf4929c849d
Parents: 16f68e3
Author: Sankar Hariappan <ma...@gmail.com>
Authored: Thu Jun 8 16:39:56 2017 -0700
Committer: Thejas M Nair <th...@hortonworks.com>
Committed: Thu Jun 8 16:40:12 2017 -0700
----------------------------------------------------------------------
.../hive/ql/parse/TestReplicationScenarios.java | 106 +
metastore/if/hive_metastore.thrift | 18 +-
.../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 2545 ++++++++++--------
.../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 134 +
.../ThriftHiveMetastore_server.skeleton.cpp | 5 +
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 387 ++-
.../gen/thrift/gen-cpp/hive_metastore_types.h | 84 +
.../hive/metastore/api/CmRecycleRequest.java | 488 ++++
.../hive/metastore/api/CmRecycleResponse.java | 283 ++
.../hive/metastore/api/ThriftHiveMetastore.java | 1507 +++++++++--
.../gen-php/metastore/ThriftHiveMetastore.php | 245 ++
.../src/gen/thrift/gen-php/metastore/Types.php | 148 +
.../hive_metastore/ThriftHiveMetastore-remote | 7 +
.../hive_metastore/ThriftHiveMetastore.py | 208 ++
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 128 +
.../gen/thrift/gen-rb/hive_metastore_types.rb | 35 +
.../gen/thrift/gen-rb/thrift_hive_metastore.rb | 61 +
.../hadoop/hive/metastore/HiveMetaStore.java | 6 +
.../hive/metastore/HiveMetaStoreClient.java | 12 +
.../hadoop/hive/metastore/IMetaStoreClient.java | 12 +-
.../apache/hadoop/hive/metastore/Warehouse.java | 5 +
.../apache/hadoop/hive/ql/metadata/Hive.java | 26 +-
22 files changed, 5003 insertions(+), 1447 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a1869692/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index 766d858..4d65fe0 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -1412,6 +1412,112 @@ public class TestReplicationScenarios {
}
@Test
+ public void testInsertOverwriteOnUnpartitionedTableWithCM() throws IOException {
+ String testName = "insertOverwriteOnUnpartitionedTableWithCM";
+ LOG.info("Testing " + testName);
+ String dbName = testName + "_" + tid;
+
+ run("CREATE DATABASE " + dbName);
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
+
+ advanceDumpDir();
+ run("REPL DUMP " + dbName);
+ String replDumpLocn = getResult(0, 0);
+ String replDumpId = getResult(0, 1, true);
+ LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+
+ // After INSERT INTO operation, get the last Repl ID
+ String[] unptn_data = new String[] { "thirteen" };
+ run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')");
+ run("REPL DUMP " + dbName + " FROM " + replDumpId);
+ String insertDumpId = getResult(0, 1, false);
+
+ // Insert overwrite on unpartitioned table
+ String[] data_after_ovwrite = new String[] { "hundred" };
+ run("INSERT OVERWRITE TABLE " + dbName + ".unptned values('" + data_after_ovwrite[0] + "')");
+
+ // Dump only one INSERT INTO operation on the table.
+ advanceDumpDir();
+ run("REPL DUMP " + dbName + " FROM " + replDumpId + " TO " + insertDumpId);
+ String incrementalDumpLocn = getResult(0, 0);
+ String incrementalDumpId = getResult(0, 1, true);
+ LOG.info("Incremental-Dump: Dumped to {} with id {} from {}", incrementalDumpLocn, incrementalDumpId, replDumpId);
+ replDumpId = incrementalDumpId;
+
+ // After Load from this dump, all target tables/partitions will have initial set of data but source will have latest data.
+ run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
+ verifyRun("SELECT a from " + dbName + "_dupe.unptned ORDER BY a", unptn_data);
+
+ // Dump the remaining INSERT OVERWRITE operations on the table.
+ advanceDumpDir();
+ run("REPL DUMP " + dbName + " FROM " + replDumpId);
+ incrementalDumpLocn = getResult(0, 0);
+ incrementalDumpId = getResult(0, 1, true);
+ LOG.info("Incremental-Dump: Dumped to {} with id {} from {}", incrementalDumpLocn, incrementalDumpId, replDumpId);
+
+ // After load, shall see the overwritten data.
+ run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
+ verifyRun("SELECT a from " + dbName + "_dupe.unptned ORDER BY a", data_after_ovwrite);
+ }
+
+ @Test
+ public void testInsertOverwriteOnPartitionedTableWithCM() throws IOException {
+ String testName = "insertOverwriteOnPartitionedTableWithCM";
+ LOG.info("Testing " + testName);
+ String dbName = testName + "_" + tid;
+
+ run("CREATE DATABASE " + dbName);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE");
+
+ advanceDumpDir();
+ run("REPL DUMP " + dbName);
+ String replDumpLocn = getResult(0, 0);
+ String replDumpId = getResult(0, 1, true);
+ LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+
+ // INSERT INTO 2 partitions and get the last repl ID
+ String[] ptn_data_1 = new String[] { "fourteen" };
+ String[] ptn_data_2 = new String[] { "fifteen", "sixteen" };
+ run("INSERT INTO TABLE " + dbName + ".ptned partition(b=1) values('" + ptn_data_1[0] + "')");
+ run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[0] + "')");
+ run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[1] + "')");
+ run("REPL DUMP " + dbName + " FROM " + replDumpId);
+ String insertDumpId = getResult(0, 1, false);
+
+ // Insert overwrite on one partition with multiple files
+ String[] data_after_ovwrite = new String[] { "hundred" };
+ run("INSERT OVERWRITE TABLE " + dbName + ".ptned partition(b=2) values('" + data_after_ovwrite[0] + "')");
+ verifySetup("SELECT a from " + dbName + ".ptned where (b=2)", data_after_ovwrite);
+
+ // Dump only 2 INSERT INTO operations.
+ advanceDumpDir();
+ run("REPL DUMP " + dbName + " FROM " + replDumpId + " TO " + insertDumpId);
+ String incrementalDumpLocn = getResult(0, 0);
+ String incrementalDumpId = getResult(0, 1, true);
+ LOG.info("Incremental-Dump: Dumped to {} with id {} from {}", incrementalDumpLocn, incrementalDumpId, replDumpId);
+ replDumpId = incrementalDumpId;
+
+ // After Load from this dump, all target tables/partitions will have initial set of data.
+ run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned where (b=1) ORDER BY a", ptn_data_1);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned where (b=2) ORDER BY a", ptn_data_2);
+
+ // Dump the remaining INSERT OVERWRITE operation on the table.
+ advanceDumpDir();
+ run("REPL DUMP " + dbName + " FROM " + replDumpId);
+ incrementalDumpLocn = getResult(0, 0);
+ incrementalDumpId = getResult(0, 1, true);
+ LOG.info("Incremental-Dump: Dumped to {} with id {} from {}", incrementalDumpLocn, incrementalDumpId, replDumpId);
+
+ // After load, shall see the overwritten data.
+ run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned where (b=1) ORDER BY a", ptn_data_1);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned where (b=2) ORDER BY a", data_after_ovwrite);
+ }
+
+ @Test
public void testViewsReplication() throws IOException {
String testName = "viewsReplication";
String dbName = createDB(testName);
http://git-wip-us.apache.org/repos/asf/hive/blob/a1869692/metastore/if/hive_metastore.thrift
----------------------------------------------------------------------
diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift
index 0573f0c..042a5d8 100755
--- a/metastore/if/hive_metastore.thrift
+++ b/metastore/if/hive_metastore.thrift
@@ -985,6 +985,16 @@ struct GetTablesResult {
1: required list<Table> tables
}
+// Request type for cm_recycle
+struct CmRecycleRequest {
+ 1: required string dataPath,
+ 2: required bool purge
+}
+
+// Response type for cm_recycle
+struct CmRecycleResponse {
+}
+
struct TableMeta {
1: required string dbName;
2: required string tableName;
@@ -1136,11 +1146,8 @@ service ThriftHiveMetastore extends fb303.FacebookService
Table get_table(1:string dbname, 2:string tbl_name)
throws (1:MetaException o1, 2:NoSuchObjectException o2)
list<Table> get_table_objects_by_name(1:string dbname, 2:list<string> tbl_names)
- GetTableResult get_table_req(1:GetTableRequest req)
- throws (1:MetaException o1, 2:NoSuchObjectException o2)
+ GetTableResult get_table_req(1:GetTableRequest req) throws (1:MetaException o1, 2:NoSuchObjectException o2)
GetTablesResult get_table_objects_by_name_req(1:GetTablesRequest req)
-
-
throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3)
// Get a list of table names that match a filter.
@@ -1542,6 +1549,9 @@ service ThriftHiveMetastore extends fb303.FacebookService
FireEventResponse fire_listener_event(1:FireEventRequest rqst)
void flushCache()
+ // Repl Change Management api
+ CmRecycleResponse cm_recycle(1:CmRecycleRequest request) throws(1:MetaException o1)
+
GetFileMetadataByExprResult get_file_metadata_by_expr(1:GetFileMetadataByExprRequest req)
GetFileMetadataResult get_file_metadata(1:GetFileMetadataRequest req)
PutFileMetadataResult put_file_metadata(1:PutFileMetadataRequest req)