You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2022/04/05 13:26:38 UTC
[hive] branch master updated: HIVE-25934: Non blocking RENAME PARTITION implementation (Denys Kuzmenko, reviewed by Peter Vary and Rajesh Balamohan)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 517eb8c9e3 HIVE-25934: Non blocking RENAME PARTITION implementation (Denys Kuzmenko, reviewed by Peter Vary and Rajesh Balamohan)
517eb8c9e3 is described below
commit 517eb8c9e36537bafb72e7b4135e58517608344c
Author: Denys Kuzmenko <dk...@cloudera.com>
AuthorDate: Tue Apr 5 15:26:26 2022 +0200
HIVE-25934: Non blocking RENAME PARTITION implementation (Denys Kuzmenko, reviewed by Peter Vary and Rajesh Balamohan)
Closes #3015
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 5 +
.../rename/AlterTableRenamePartitionAnalyzer.java | 11 +-
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 34 +++-
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 10 +-
.../ql/metadata/SessionHiveMetaStoreClient.java | 4 +-
.../hadoop/hive/ql/txn/compactor/Cleaner.java | 4 +-
.../org/apache/hadoop/hive/ql/TestTxnCommands.java | 91 +++++++++
.../hadoop/hive/ql/lockmgr/TestDbTxnManager2.java | 98 ++++++++++
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 44 +++++
.../src/gen/thrift/gen-cpp/hive_metastore_types.h | 20 +-
.../hive/metastore/api/RenamePartitionRequest.java | 208 ++++++++++++++++++++-
.../gen-php/metastore/RenamePartitionRequest.php | 48 +++++
.../src/gen/thrift/gen-py/hive_metastore/ttypes.py | 26 ++-
.../src/gen/thrift/gen-rb/hive_metastore_types.rb | 6 +-
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 5 +-
.../hadoop/hive/metastore/IMetaStoreClient.java | 10 +-
.../apache/hadoop/hive/metastore/Warehouse.java | 15 ++
.../src/main/thrift/hive_metastore.thrift | 4 +-
.../hadoop/hive/metastore/AcidEventListener.java | 76 +++++---
.../apache/hadoop/hive/metastore/HMSHandler.java | 22 ++-
.../hadoop/hive/metastore/HiveAlterHandler.java | 27 ++-
.../metastore/HiveMetaStoreClientPreCatalog.java | 4 +-
22 files changed, 706 insertions(+), 66 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 605f6f5177..73bb924611 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3088,6 +3088,11 @@ public class HiveConf extends Configuration {
"Enables non-blocking DROP PARTITION operation.\n" +
"If enabled, drop for transactional tables will not delete the data directories,\n" +
"rather create a new base directory with no datafiles.\")"),
+
+ HIVE_ACID_RENAME_PARTITION_MAKE_COPY("hive.acid.renamepartition.makecopy", false,
+ "Enables non-blocking RENAME PARTITION operation.\n" +
+ "If enabled, rename for transactional tables will not rename the partition directory,\n" +
+ "rather create a copy of it under the new path.\")"),
// Configs having to do with DeltaFilesMetricReporter, which collects lists of most recently active tables
// with the most number of active/obsolete deltas.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/rename/AlterTableRenamePartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/rename/AlterTableRenamePartitionAnalyzer.java
index bc607a505f..3db3904d12 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/rename/AlterTableRenamePartitionAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/rename/AlterTableRenamePartitionAnalyzer.java
@@ -23,6 +23,8 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.ddl.DDLWork;
import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType;
@@ -31,7 +33,7 @@ import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.ddl.table.partition.PartitionUtils;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
-import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity.WriteType;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ASTNode;
@@ -64,8 +66,13 @@ public class AlterTableRenamePartitionAnalyzer extends AbstractAlterTableAnalyz
List<Map<String, String>> allPartitionSpecs = new ArrayList<>();
allPartitionSpecs.add(partitionSpec);
allPartitionSpecs.add(newPartitionSpec);
+
+ boolean clonePart = HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_RENAME_PARTITION_MAKE_COPY)
+ || HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED)
+ && AcidUtils.isTransactionalTable(table);
+
PartitionUtils.addTablePartsOutputs(db, outputs, table, allPartitionSpecs, false,
- WriteEntity.WriteType.DDL_EXCLUSIVE);
+ clonePart ? WriteType.DDL_EXCL_WRITE : WriteType.DDL_EXCLUSIVE);
AlterTableRenamePartitionDesc desc = new AlterTableRenamePartitionDesc(tableName, partitionSpec, newPartitionSpec,
null, table);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index a03ef012df..dff8b87aa2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -3131,20 +3131,13 @@ public class AcidUtils {
if (tree.getFirstChildWithType(HiveParser.TOK_ALTERTABLE_COMPACT) != null){
return TxnType.COMPACTION;
}
- // check if soft delete
- if ((tree.getToken().getType() == HiveParser.TOK_DROPTABLE || tree.getToken().getType() == HiveParser.TOK_DROP_MATERIALIZED_VIEW)
- && (HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_CREATE_TABLE_USE_SUFFIX)
- || HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED))){
- return TxnType.SOFT_DELETE;
- }
- if (tree.getToken().getType() == HiveParser.TOK_ALTERTABLE_DROPPARTS
- && (HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_DROP_PARTITION_USE_BASE)
- || HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED))) {
+ // check if soft delete txn
+ if (isSoftDeleteTxn(conf, tree)) {
return TxnType.SOFT_DELETE;
}
return TxnType.DEFAULT;
}
-
+
private static boolean isReadOnlyTxn(ASTNode tree) {
final ASTSearcher astSearcher = new ASTSearcher();
return READ_TXN_TOKENS.contains(tree.getToken().getType())
@@ -3153,6 +3146,27 @@ public class AcidUtils {
new int[]{HiveParser.TOK_INSERT, HiveParser.TOK_TAB})
.noneMatch(pattern -> astSearcher.simpleBreadthFirstSearch(tree, pattern) != null));
}
+
+ private static boolean isSoftDeleteTxn(Configuration conf, ASTNode tree) {
+ boolean locklessReadsEnabled = HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED);
+
+ switch (tree.getToken().getType()) {
+ case HiveParser.TOK_DROPTABLE:
+ case HiveParser.TOK_DROP_MATERIALIZED_VIEW:
+ return locklessReadsEnabled
+ || HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_CREATE_TABLE_USE_SUFFIX);
+
+ case HiveParser.TOK_ALTERTABLE_DROPPARTS:
+ return locklessReadsEnabled
+ || HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_DROP_PARTITION_USE_BASE);
+
+ case HiveParser.TOK_ALTERTABLE_RENAMEPART:
+ return locklessReadsEnabled
+ || HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_RENAME_PARTITION_MAKE_COPY);
+ default:
+ return false;
+ }
+ }
@VisibleForTesting
public static void initDirCache(int durationInMts) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index f211ddcf62..fcb7f037b5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1162,6 +1162,9 @@ public class Hive {
}
}
String validWriteIds = null;
+ boolean clonePart = false;
+ long txnId = 0;
+
if (AcidUtils.isTransactionalTable(tbl)) {
TableSnapshot tableSnapshot;
if (replWriteId > 0) {
@@ -1182,11 +1185,16 @@ public class Hive {
newPart.getTPartition().setWriteId(tableSnapshot.getWriteId());
validWriteIds = tableSnapshot.getValidWriteIdList();
}
+ clonePart = HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_RENAME_PARTITION_MAKE_COPY)
+ || HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED);
+
+ txnId = Optional.ofNullable(SessionState.get())
+ .map(ss -> ss.getTxnMgr().getCurrentTxnId()).orElse(0L);
}
String catName = (tbl.getCatalogName() != null) ? tbl.getCatalogName() : getDefaultCatalog(conf);
getMSC().renamePartition(catName, tbl.getDbName(), tbl.getTableName(), pvals,
- newPart.getTPartition(), validWriteIds);
+ newPart.getTPartition(), validWriteIds, txnId, clonePart);
} catch (InvalidOperationException e){
throw new HiveException("Unable to rename partition. " + e.getMessage(), e);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index 75d36092f8..07267c8dd4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -1534,10 +1534,10 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClientWithLocalCach
@Override
public void renamePartition(String catName, String dbname, String tableName, List<String> partitionVals,
- Partition newPart, String validWriteIds) throws TException {
+ Partition newPart, String validWriteIds, long txnId, boolean makeCopy) throws TException {
org.apache.hadoop.hive.metastore.api.Table table = getTempTable(dbname, tableName);
if (table == null) {
- super.renamePartition(catName, dbname, tableName, partitionVals, newPart, validWriteIds);
+ super.renamePartition(catName, dbname, tableName, partitionVals, newPart, validWriteIds, txnId, makeCopy);
return;
}
TempTable tt = getPartitionedTempTable(table);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
index 11652d0a91..478ec03278 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
@@ -352,7 +352,7 @@ public class Cleaner extends MetaStoreCompactorThread {
try {
res = txnHandler.lock(lockRequest);
if (res.getState() == LockState.ACQUIRED) {
- //check if partition wasn't recreated
+ //check if partition wasn't re-created
if (resolvePartition(ci) == null) {
return removeFiles(location, ci);
}
@@ -360,7 +360,7 @@ public class Cleaner extends MetaStoreCompactorThread {
} catch (NoSuchTxnException | TxnAbortedException e) {
LOG.error(e.getMessage());
} finally {
- if (res != null && res.getState() != LockState.NOT_ACQUIRED) {
+ if (res != null) {
try {
txnHandler.unlock(new UnlockRequest(res.getLockid()));
} catch (NoSuchLockException | TxnOpenException e) {
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 1092103181..fa01592a43 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -111,6 +111,7 @@ public class TestTxnCommands extends TxnCommandsBaseForTests {
//of these tests.
HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, false);
HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_ACID_DROP_PARTITION_USE_BASE, false);
+ HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_ACID_RENAME_PARTITION_MAKE_COPY, false);
HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_ACID_CREATE_TABLE_USE_SUFFIX, false);
HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE, false);
}
@@ -1861,4 +1862,94 @@ public class TestTxnCommands extends TxnCommandsBaseForTests {
Assert.assertEquals("Unexpected number of compactions in history", 0, resp.getCompactsSize());
}
}
+
+ @Test
+ public void testRenameMakeCopyPartition() throws Exception {
+ runStatementOnDriver("insert into " + Table.ACIDTBLPART + " partition (p='a') values (1,2),(3,4)");
+ runStatementOnDriver("insert into " + Table.ACIDTBLPART + " partition (p='b') values (5,5),(4,4)");
+
+ HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_ACID_RENAME_PARTITION_MAKE_COPY, true);
+ runStatementOnDriver("alter table " + Table.ACIDTBLPART + " partition (p='b') rename to partition (p='c')");
+
+ FileSystem fs = FileSystem.get(hiveConf);
+ FileStatus[] stat = fs.listStatus(new Path(getWarehouseDir(), Table.ACIDTBLPART.toString().toLowerCase() + "/p=b"),
+ AcidUtils.baseFileFilter);
+ if (1 != stat.length) {
+ Assert.fail("Expecting 1 base and found " + stat.length + " files " + Arrays.toString(stat));
+ }
+ String name = stat[0].getPath().getName();
+ Assert.assertEquals("base_0000003", name);
+ stat = fs.listStatus(new Path(getWarehouseDir(), Table.ACIDTBLPART.toString().toLowerCase() + "/p=a"),
+ AcidUtils.baseFileFilter);
+ if (0 != stat.length) {
+ Assert.fail("Expecting no base and found " + stat.length + " files " + Arrays.toString(stat));
+ }
+
+ List<String> r = runStatementOnDriver("select * from " + Table.ACIDTBLPART + " where p='b'");
+ Assert.assertEquals(0, r.size());
+
+ r = runStatementOnDriver("select * from " + Table.ACIDTBLPART);
+ Assert.assertEquals(4, r.size());
+
+ TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+ ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest());
+
+ Assert.assertEquals("Unexpected number of compactions in history", 1, resp.getCompactsSize());
+ Assert.assertTrue(resp.getCompacts().stream().anyMatch(
+ ci -> TxnStore.CLEANING_RESPONSE.equals(ci.getState()) && "p=b".equals(ci.getPartitionname())));
+
+ runCleaner(hiveConf);
+
+ stat = fs.listStatus(new Path(getWarehouseDir(), Table.ACIDTBLPART.toString().toLowerCase()),
+ path -> path.getName().equals("p=b"));
+ if (0 != stat.length) {
+ Assert.fail("Expecting partition data to be removed from FS");
+ }
+ }
+
+ @Test
+ public void testRenameMakeCopyNestedPartition() throws Exception {
+ runStatementOnDriver("insert into " + Table.ACIDTBLNESTEDPART + " partition (p1='a', p2='b', p3='c') values (1,1),(2,2)");
+ runStatementOnDriver("insert into " + Table.ACIDTBLNESTEDPART + " partition (p1='a', p2='b', p3='d') values (3,3),(4,4)");
+
+ HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_ACID_RENAME_PARTITION_MAKE_COPY, true);
+ runStatementOnDriver("alter table " + Table.ACIDTBLNESTEDPART + " partition (p1='a', p2='b', p3='d')" +
+ " rename to partition (p1='a', p2='c', p3='d')");
+
+ TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+ ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest());
+ Assert.assertEquals("Unexpected number of compactions in history", 1, resp.getCompactsSize());
+
+ FileSystem fs = FileSystem.get(hiveConf);
+ FileStatus[] stat;
+
+ String partName = "p1=a/p2=b/p3=d";
+ Assert.assertTrue(resp.getCompacts().stream().anyMatch(
+ ci -> TxnStore.CLEANING_RESPONSE.equals(ci.getState()) && partName.equals(ci.getPartitionname())));
+
+ stat = fs.listStatus(new Path(getWarehouseDir(), Table.ACIDTBLNESTEDPART.toString().toLowerCase() + "/" + partName),
+ AcidUtils.baseFileFilter);
+ if (1 != stat.length) {
+ Assert.fail("Expecting 1 base and found " + stat.length + " files " + Arrays.toString(stat));
+ }
+ String name = stat[0].getPath().getName();
+ Assert.assertEquals("base_0000003", name);
+
+ stat = fs.listStatus(new Path(getWarehouseDir(), Table.ACIDTBLNESTEDPART.toString().toLowerCase() + "/p1=a/p2=c/p3=d"),
+ AcidUtils.baseFileFilter);
+ if (0 != stat.length) {
+ Assert.fail("Expecting no base and found " + stat.length + " files " + Arrays.toString(stat));
+ }
+
+ List<String> r = runStatementOnDriver("select * from " + Table.ACIDTBLNESTEDPART);
+ Assert.assertEquals(4, r.size());
+
+ runCleaner(hiveConf);
+
+ stat = fs.listStatus(new Path(getWarehouseDir(), Table.ACIDTBLNESTEDPART.toString().toLowerCase() + "/p1=a/p2=b"),
+ path -> path.getName().equals("p3=d"));
+ if (0 != stat.length) {
+ Assert.fail("Expecting partition data to be removed from FS");
+ }
+ }
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index e6dd859cbf..1ec94df008 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -3816,4 +3816,102 @@ public class TestDbTxnManager2 extends DbTxnManagerEndToEndTestBase{
Assert.assertEquals("Expecting 2 rows and found " + res.size(), 2, res.size());
driver.run("drop materialized view mv_tab_acid");
}
+
+ @Test
+ public void testRenamePartitionNonBlocking() throws Exception {
+ testRenamePartition(false);
+ }
+ @Test
+ public void testRenamePartitionBlocking() throws Exception {
+ testRenamePartition(true);
+ }
+
+ private void testRenamePartition(boolean blocking) throws Exception {
+ dropTable(new String[] {"tab_acid"});
+ FileSystem fs = FileSystem.get(conf);
+
+ HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_LOCKS_PARTITION_THRESHOLD, 1);
+ driver = Mockito.spy(driver);
+
+ HiveConf.setBoolVar(driver2.getConf(), HiveConf.ConfVars.HIVE_ACID_RENAME_PARTITION_MAKE_COPY, !blocking);
+ driver2 = Mockito.spy(driver2);
+
+ driver.run("create table if not exists tab_acid (a int, b int) partitioned by (p string) " +
+ "stored as orc TBLPROPERTIES ('transactional'='true')");
+ driver.run("insert into tab_acid partition(p) (a,b,p) values(1,2,'foo'),(3,4,'bar')");
+
+ driver.compileAndRespond("select * from tab_acid");
+ List<String> res = new ArrayList<>();
+
+ driver.lockAndRespond();
+ List<ShowLocksResponseElement> locks = getLocks();
+ Assert.assertEquals("Unexpected lock count", 1, locks.size());
+
+ checkLock(LockType.SHARED_READ,
+ LockState.ACQUIRED, "default", "tab_acid", null, locks);
+
+ DbTxnManager txnMgr2 = (DbTxnManager) TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ swapTxnManager(txnMgr2);
+ driver2.compileAndRespond("alter table tab_acid partition (p='foo') rename to partition (p='baz')");
+
+ if (blocking) {
+ txnMgr2.acquireLocks(driver2.getPlan(), ctx, null, false);
+ locks = getLocks();
+
+ ShowLocksResponseElement checkLock = checkLock(LockType.EXCLUSIVE,
+ LockState.WAITING, "default", "tab_acid", "p=foo", locks);
+
+ swapTxnManager(txnMgr);
+ Mockito.doNothing().when(driver).lockAndRespond();
+ driver.run();
+
+ driver.getFetchTask().fetch(res);
+ swapTxnManager(txnMgr2);
+
+ FieldSetter.setField(txnMgr2, txnMgr2.getClass().getDeclaredField("numStatements"), 0);
+ txnMgr2.getMS().unlock(checkLock.getLockid());
+ }
+ driver2.lockAndRespond();
+ locks = getLocks();
+ Assert.assertEquals("Unexpected lock count", blocking ? 1 : 2, locks.size());
+
+ checkLock(blocking ? LockType.EXCLUSIVE : LockType.EXCL_WRITE,
+ LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks);
+
+ Mockito.doNothing().when(driver2).lockAndRespond();
+ driver2.run();
+
+ if (!blocking) {
+ swapTxnManager(txnMgr);
+ Mockito.doNothing().when(driver).lockAndRespond();
+ driver.run();
+ }
+ Mockito.reset(driver, driver2);
+
+ FileStatus[] stat = fs.listStatus(new Path(getWarehouseDir(), "tab_acid" + (blocking ? "" : "/p=foo")),
+ (blocking ? path -> path.getName().equals("p=foo") : AcidUtils.baseFileFilter));
+ if ((blocking ? 0 : 1) != stat.length) {
+ Assert.fail("Partition data was " + (blocking ? "not " : "") + "removed from FS");
+ }
+ driver.getFetchTask().fetch(res);
+ Assert.assertEquals("Expecting 2 rows and found " + res.size(), 2, res.size());
+
+ driver.run("select * from tab_acid where p='foo'");
+ res = new ArrayList<>();
+ driver.getFetchTask().fetch(res);
+ Assert.assertEquals("Expecting 0 rows and found " + res.size(), 0, res.size());
+
+ driver.run("select * from tab_acid");
+ res = new ArrayList<>();
+ driver.getFetchTask().fetch(res);
+ Assert.assertEquals("Expecting 2 rows and found " + res.size(), 2, res.size());
+
+ //re-create partition with the same name
+ driver.run("insert into tab_acid partition(p) (a,b,p) values(1,2,'foo')");
+
+ driver.run("select * from tab_acid where p='foo'");
+ res = new ArrayList<>();
+ driver.getFetchTask().fetch(res);
+ Assert.assertEquals("Expecting 1 rows and found " + res.size(), 1, res.size());
+ }
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index 3952126014..a5bd17ff46 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@ -44441,6 +44441,16 @@ void RenamePartitionRequest::__set_validWriteIdList(const std::string& val) {
this->validWriteIdList = val;
__isset.validWriteIdList = true;
}
+
+void RenamePartitionRequest::__set_txnId(const int64_t val) {
+ this->txnId = val;
+__isset.txnId = true;
+}
+
+void RenamePartitionRequest::__set_clonePart(const bool val) {
+ this->clonePart = val;
+__isset.clonePart = true;
+}
std::ostream& operator<<(std::ostream& out, const RenamePartitionRequest& obj)
{
obj.printTo(out);
@@ -44533,6 +44543,22 @@ uint32_t RenamePartitionRequest::read(::apache::thrift::protocol::TProtocol* ipr
xfer += iprot->skip(ftype);
}
break;
+ case 7:
+ if (ftype == ::apache::thrift::protocol::T_I64) {
+ xfer += iprot->readI64(this->txnId);
+ this->__isset.txnId = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 8:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->clonePart);
+ this->__isset.clonePart = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
default:
xfer += iprot->skip(ftype);
break;
@@ -44592,6 +44618,16 @@ uint32_t RenamePartitionRequest::write(::apache::thrift::protocol::TProtocol* op
xfer += oprot->writeString(this->validWriteIdList);
xfer += oprot->writeFieldEnd();
}
+ if (this->__isset.txnId) {
+ xfer += oprot->writeFieldBegin("txnId", ::apache::thrift::protocol::T_I64, 7);
+ xfer += oprot->writeI64(this->txnId);
+ xfer += oprot->writeFieldEnd();
+ }
+ if (this->__isset.clonePart) {
+ xfer += oprot->writeFieldBegin("clonePart", ::apache::thrift::protocol::T_BOOL, 8);
+ xfer += oprot->writeBool(this->clonePart);
+ xfer += oprot->writeFieldEnd();
+ }
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -44605,6 +44641,8 @@ void swap(RenamePartitionRequest &a, RenamePartitionRequest &b) {
swap(a.partVals, b.partVals);
swap(a.newPart, b.newPart);
swap(a.validWriteIdList, b.validWriteIdList);
+ swap(a.txnId, b.txnId);
+ swap(a.clonePart, b.clonePart);
swap(a.__isset, b.__isset);
}
@@ -44615,6 +44653,8 @@ RenamePartitionRequest::RenamePartitionRequest(const RenamePartitionRequest& oth
partVals = other1560.partVals;
newPart = other1560.newPart;
validWriteIdList = other1560.validWriteIdList;
+ txnId = other1560.txnId;
+ clonePart = other1560.clonePart;
__isset = other1560.__isset;
}
RenamePartitionRequest& RenamePartitionRequest::operator=(const RenamePartitionRequest& other1561) {
@@ -44624,6 +44664,8 @@ RenamePartitionRequest& RenamePartitionRequest::operator=(const RenamePartitionR
partVals = other1561.partVals;
newPart = other1561.newPart;
validWriteIdList = other1561.validWriteIdList;
+ txnId = other1561.txnId;
+ clonePart = other1561.clonePart;
__isset = other1561.__isset;
return *this;
}
@@ -44636,6 +44678,8 @@ void RenamePartitionRequest::printTo(std::ostream& out) const {
out << ", " << "partVals=" << to_string(partVals);
out << ", " << "newPart=" << to_string(newPart);
out << ", " << "validWriteIdList="; (__isset.validWriteIdList ? (out << to_string(validWriteIdList)) : (out << "<null>"));
+ out << ", " << "txnId="; (__isset.txnId ? (out << to_string(txnId)) : (out << "<null>"));
+ out << ", " << "clonePart="; (__isset.clonePart ? (out << to_string(clonePart)) : (out << "<null>"));
out << ")";
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
index b1bf6039f2..69a14bd9a6 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
@@ -16832,9 +16832,11 @@ void swap(AlterPartitionsResponse &a, AlterPartitionsResponse &b);
std::ostream& operator<<(std::ostream& out, const AlterPartitionsResponse& obj);
typedef struct _RenamePartitionRequest__isset {
- _RenamePartitionRequest__isset() : catName(false), validWriteIdList(false) {}
+ _RenamePartitionRequest__isset() : catName(false), validWriteIdList(false), txnId(false), clonePart(false) {}
bool catName :1;
bool validWriteIdList :1;
+ bool txnId :1;
+ bool clonePart :1;
} _RenamePartitionRequest__isset;
class RenamePartitionRequest : public virtual ::apache::thrift::TBase {
@@ -16842,7 +16844,7 @@ class RenamePartitionRequest : public virtual ::apache::thrift::TBase {
RenamePartitionRequest(const RenamePartitionRequest&);
RenamePartitionRequest& operator=(const RenamePartitionRequest&);
- RenamePartitionRequest() : catName(), dbName(), tableName(), validWriteIdList() {
+ RenamePartitionRequest() : catName(), dbName(), tableName(), validWriteIdList(), txnId(0), clonePart(0) {
}
virtual ~RenamePartitionRequest() noexcept;
@@ -16852,6 +16854,8 @@ class RenamePartitionRequest : public virtual ::apache::thrift::TBase {
std::vector<std::string> partVals;
Partition newPart;
std::string validWriteIdList;
+ int64_t txnId;
+ bool clonePart;
_RenamePartitionRequest__isset __isset;
@@ -16867,6 +16871,10 @@ class RenamePartitionRequest : public virtual ::apache::thrift::TBase {
void __set_validWriteIdList(const std::string& val);
+ void __set_txnId(const int64_t val);
+
+ void __set_clonePart(const bool val);
+
bool operator == (const RenamePartitionRequest & rhs) const
{
if (__isset.catName != rhs.__isset.catName)
@@ -16885,6 +16893,14 @@ class RenamePartitionRequest : public virtual ::apache::thrift::TBase {
return false;
else if (__isset.validWriteIdList && !(validWriteIdList == rhs.validWriteIdList))
return false;
+ if (__isset.txnId != rhs.__isset.txnId)
+ return false;
+ else if (__isset.txnId && !(txnId == rhs.txnId))
+ return false;
+ if (__isset.clonePart != rhs.__isset.clonePart)
+ return false;
+ else if (__isset.clonePart && !(clonePart == rhs.clonePart))
+ return false;
return true;
}
bool operator != (const RenamePartitionRequest &rhs) const {
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/RenamePartitionRequest.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/RenamePartitionRequest.java
index 43f8120a3f..217116d5ed 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/RenamePartitionRequest.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/RenamePartitionRequest.java
@@ -17,6 +17,8 @@ package org.apache.hadoop.hive.metastore.api;
private static final org.apache.thrift.protocol.TField PART_VALS_FIELD_DESC = new org.apache.thrift.protocol.TField("partVals", org.apache.thrift.protocol.TType.LIST, (short)4);
private static final org.apache.thrift.protocol.TField NEW_PART_FIELD_DESC = new org.apache.thrift.protocol.TField("newPart", org.apache.thrift.protocol.TType.STRUCT, (short)5);
private static final org.apache.thrift.protocol.TField VALID_WRITE_ID_LIST_FIELD_DESC = new org.apache.thrift.protocol.TField("validWriteIdList", org.apache.thrift.protocol.TType.STRING, (short)6);
+ private static final org.apache.thrift.protocol.TField TXN_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("txnId", org.apache.thrift.protocol.TType.I64, (short)7);
+ private static final org.apache.thrift.protocol.TField CLONE_PART_FIELD_DESC = new org.apache.thrift.protocol.TField("clonePart", org.apache.thrift.protocol.TType.BOOL, (short)8);
private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new RenamePartitionRequestStandardSchemeFactory();
private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new RenamePartitionRequestTupleSchemeFactory();
@@ -27,6 +29,8 @@ package org.apache.hadoop.hive.metastore.api;
private @org.apache.thrift.annotation.Nullable java.util.List<java.lang.String> partVals; // required
private @org.apache.thrift.annotation.Nullable Partition newPart; // required
private @org.apache.thrift.annotation.Nullable java.lang.String validWriteIdList; // optional
+ private long txnId; // optional
+ private boolean clonePart; // optional
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -35,7 +39,9 @@ package org.apache.hadoop.hive.metastore.api;
TABLE_NAME((short)3, "tableName"),
PART_VALS((short)4, "partVals"),
NEW_PART((short)5, "newPart"),
- VALID_WRITE_ID_LIST((short)6, "validWriteIdList");
+ VALID_WRITE_ID_LIST((short)6, "validWriteIdList"),
+ TXN_ID((short)7, "txnId"),
+ CLONE_PART((short)8, "clonePart");
private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
@@ -63,6 +69,10 @@ package org.apache.hadoop.hive.metastore.api;
return NEW_PART;
case 6: // VALID_WRITE_ID_LIST
return VALID_WRITE_ID_LIST;
+ case 7: // TXN_ID
+ return TXN_ID;
+ case 8: // CLONE_PART
+ return CLONE_PART;
default:
return null;
}
@@ -104,7 +114,10 @@ package org.apache.hadoop.hive.metastore.api;
}
// isset id assignments
- private static final _Fields optionals[] = {_Fields.CAT_NAME,_Fields.VALID_WRITE_ID_LIST};
+ private static final int __TXNID_ISSET_ID = 0;
+ private static final int __CLONEPART_ISSET_ID = 1;
+ private byte __isset_bitfield = 0;
+ private static final _Fields optionals[] = {_Fields.CAT_NAME,_Fields.VALID_WRITE_ID_LIST,_Fields.TXN_ID,_Fields.CLONE_PART};
public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -121,6 +134,10 @@ package org.apache.hadoop.hive.metastore.api;
new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Partition.class)));
tmpMap.put(_Fields.VALID_WRITE_ID_LIST, new org.apache.thrift.meta_data.FieldMetaData("validWriteIdList", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ tmpMap.put(_Fields.TXN_ID, new org.apache.thrift.meta_data.FieldMetaData("txnId", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
+ tmpMap.put(_Fields.CLONE_PART, new org.apache.thrift.meta_data.FieldMetaData("clonePart", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(RenamePartitionRequest.class, metaDataMap);
}
@@ -145,6 +162,7 @@ package org.apache.hadoop.hive.metastore.api;
* Performs a deep copy on <i>other</i>.
*/
public RenamePartitionRequest(RenamePartitionRequest other) {
+ __isset_bitfield = other.__isset_bitfield;
if (other.isSetCatName()) {
this.catName = other.catName;
}
@@ -164,6 +182,8 @@ package org.apache.hadoop.hive.metastore.api;
if (other.isSetValidWriteIdList()) {
this.validWriteIdList = other.validWriteIdList;
}
+ this.txnId = other.txnId;
+ this.clonePart = other.clonePart;
}
public RenamePartitionRequest deepCopy() {
@@ -178,6 +198,10 @@ package org.apache.hadoop.hive.metastore.api;
this.partVals = null;
this.newPart = null;
this.validWriteIdList = null;
+ setTxnIdIsSet(false);
+ this.txnId = 0;
+ setClonePartIsSet(false);
+ this.clonePart = false;
}
@org.apache.thrift.annotation.Nullable
@@ -340,6 +364,50 @@ package org.apache.hadoop.hive.metastore.api;
}
}
+ public long getTxnId() {
+ return this.txnId;
+ }
+
+ public void setTxnId(long txnId) {
+ this.txnId = txnId;
+ setTxnIdIsSet(true);
+ }
+
+ public void unsetTxnId() {
+ __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __TXNID_ISSET_ID);
+ }
+
+ /** Returns true if field txnId is set (has been assigned a value) and false otherwise */
+ public boolean isSetTxnId() {
+ return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __TXNID_ISSET_ID);
+ }
+
+ public void setTxnIdIsSet(boolean value) {
+ __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __TXNID_ISSET_ID, value);
+ }
+
+ public boolean isClonePart() {
+ return this.clonePart;
+ }
+
+ public void setClonePart(boolean clonePart) {
+ this.clonePart = clonePart;
+ setClonePartIsSet(true);
+ }
+
+ public void unsetClonePart() {
+ __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __CLONEPART_ISSET_ID);
+ }
+
+ /** Returns true if field clonePart is set (has been assigned a value) and false otherwise */
+ public boolean isSetClonePart() {
+ return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __CLONEPART_ISSET_ID);
+ }
+
+ public void setClonePartIsSet(boolean value) {
+ __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __CLONEPART_ISSET_ID, value);
+ }
+
public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) {
switch (field) {
case CAT_NAME:
@@ -390,6 +458,22 @@ package org.apache.hadoop.hive.metastore.api;
}
break;
+ case TXN_ID:
+ if (value == null) {
+ unsetTxnId();
+ } else {
+ setTxnId((java.lang.Long)value);
+ }
+ break;
+
+ case CLONE_PART:
+ if (value == null) {
+ unsetClonePart();
+ } else {
+ setClonePart((java.lang.Boolean)value);
+ }
+ break;
+
}
}
@@ -414,6 +498,12 @@ package org.apache.hadoop.hive.metastore.api;
case VALID_WRITE_ID_LIST:
return getValidWriteIdList();
+ case TXN_ID:
+ return getTxnId();
+
+ case CLONE_PART:
+ return isClonePart();
+
}
throw new java.lang.IllegalStateException();
}
@@ -437,6 +527,10 @@ package org.apache.hadoop.hive.metastore.api;
return isSetNewPart();
case VALID_WRITE_ID_LIST:
return isSetValidWriteIdList();
+ case TXN_ID:
+ return isSetTxnId();
+ case CLONE_PART:
+ return isSetClonePart();
}
throw new java.lang.IllegalStateException();
}
@@ -508,6 +602,24 @@ package org.apache.hadoop.hive.metastore.api;
return false;
}
+ boolean this_present_txnId = true && this.isSetTxnId();
+ boolean that_present_txnId = true && that.isSetTxnId();
+ if (this_present_txnId || that_present_txnId) {
+ if (!(this_present_txnId && that_present_txnId))
+ return false;
+ if (this.txnId != that.txnId)
+ return false;
+ }
+
+ boolean this_present_clonePart = true && this.isSetClonePart();
+ boolean that_present_clonePart = true && that.isSetClonePart();
+ if (this_present_clonePart || that_present_clonePart) {
+ if (!(this_present_clonePart && that_present_clonePart))
+ return false;
+ if (this.clonePart != that.clonePart)
+ return false;
+ }
+
return true;
}
@@ -539,6 +651,14 @@ package org.apache.hadoop.hive.metastore.api;
if (isSetValidWriteIdList())
hashCode = hashCode * 8191 + validWriteIdList.hashCode();
+ hashCode = hashCode * 8191 + ((isSetTxnId()) ? 131071 : 524287);
+ if (isSetTxnId())
+ hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(txnId);
+
+ hashCode = hashCode * 8191 + ((isSetClonePart()) ? 131071 : 524287);
+ if (isSetClonePart())
+ hashCode = hashCode * 8191 + ((clonePart) ? 131071 : 524287);
+
return hashCode;
}
@@ -610,6 +730,26 @@ package org.apache.hadoop.hive.metastore.api;
return lastComparison;
}
}
+ lastComparison = java.lang.Boolean.compare(isSetTxnId(), other.isSetTxnId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetTxnId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.txnId, other.txnId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = java.lang.Boolean.compare(isSetClonePart(), other.isSetClonePart());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetClonePart()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.clonePart, other.clonePart);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -682,6 +822,18 @@ package org.apache.hadoop.hive.metastore.api;
}
first = false;
}
+ if (isSetTxnId()) {
+ if (!first) sb.append(", ");
+ sb.append("txnId:");
+ sb.append(this.txnId);
+ first = false;
+ }
+ if (isSetClonePart()) {
+ if (!first) sb.append(", ");
+ sb.append("clonePart:");
+ sb.append(this.clonePart);
+ first = false;
+ }
sb.append(")");
return sb.toString();
}
@@ -720,6 +872,8 @@ package org.apache.hadoop.hive.metastore.api;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException {
try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
@@ -803,6 +957,22 @@ package org.apache.hadoop.hive.metastore.api;
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
+ case 7: // TXN_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.txnId = iprot.readI64();
+ struct.setTxnIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 8: // CLONE_PART
+ if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
+ struct.clonePart = iprot.readBool();
+ struct.setClonePartIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -857,6 +1027,16 @@ package org.apache.hadoop.hive.metastore.api;
oprot.writeFieldEnd();
}
}
+ if (struct.isSetTxnId()) {
+ oprot.writeFieldBegin(TXN_ID_FIELD_DESC);
+ oprot.writeI64(struct.txnId);
+ oprot.writeFieldEnd();
+ }
+ if (struct.isSetClonePart()) {
+ oprot.writeFieldBegin(CLONE_PART_FIELD_DESC);
+ oprot.writeBool(struct.clonePart);
+ oprot.writeFieldEnd();
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -891,13 +1071,25 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetValidWriteIdList()) {
optionals.set(1);
}
- oprot.writeBitSet(optionals, 2);
+ if (struct.isSetTxnId()) {
+ optionals.set(2);
+ }
+ if (struct.isSetClonePart()) {
+ optionals.set(3);
+ }
+ oprot.writeBitSet(optionals, 4);
if (struct.isSetCatName()) {
oprot.writeString(struct.catName);
}
if (struct.isSetValidWriteIdList()) {
oprot.writeString(struct.validWriteIdList);
}
+ if (struct.isSetTxnId()) {
+ oprot.writeI64(struct.txnId);
+ }
+ if (struct.isSetClonePart()) {
+ oprot.writeBool(struct.clonePart);
+ }
}
@Override
@@ -921,7 +1113,7 @@ package org.apache.hadoop.hive.metastore.api;
struct.newPart = new Partition();
struct.newPart.read(iprot);
struct.setNewPartIsSet(true);
- java.util.BitSet incoming = iprot.readBitSet(2);
+ java.util.BitSet incoming = iprot.readBitSet(4);
if (incoming.get(0)) {
struct.catName = iprot.readString();
struct.setCatNameIsSet(true);
@@ -930,6 +1122,14 @@ package org.apache.hadoop.hive.metastore.api;
struct.validWriteIdList = iprot.readString();
struct.setValidWriteIdListIsSet(true);
}
+ if (incoming.get(2)) {
+ struct.txnId = iprot.readI64();
+ struct.setTxnIdIsSet(true);
+ }
+ if (incoming.get(3)) {
+ struct.clonePart = iprot.readBool();
+ struct.setClonePartIsSet(true);
+ }
}
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/RenamePartitionRequest.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/RenamePartitionRequest.php
index 260f93ed28..2411d0eb0c 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/RenamePartitionRequest.php
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/RenamePartitionRequest.php
@@ -56,6 +56,16 @@ class RenamePartitionRequest
'isRequired' => false,
'type' => TType::STRING,
),
+ 7 => array(
+ 'var' => 'txnId',
+ 'isRequired' => false,
+ 'type' => TType::I64,
+ ),
+ 8 => array(
+ 'var' => 'clonePart',
+ 'isRequired' => false,
+ 'type' => TType::BOOL,
+ ),
);
/**
@@ -82,6 +92,14 @@ class RenamePartitionRequest
* @var string
*/
public $validWriteIdList = null;
+ /**
+ * @var int
+ */
+ public $txnId = null;
+ /**
+ * @var bool
+ */
+ public $clonePart = null;
public function __construct($vals = null)
{
@@ -104,6 +122,12 @@ class RenamePartitionRequest
if (isset($vals['validWriteIdList'])) {
$this->validWriteIdList = $vals['validWriteIdList'];
}
+ if (isset($vals['txnId'])) {
+ $this->txnId = $vals['txnId'];
+ }
+ if (isset($vals['clonePart'])) {
+ $this->clonePart = $vals['clonePart'];
+ }
}
}
@@ -178,6 +202,20 @@ class RenamePartitionRequest
$xfer += $input->skip($ftype);
}
break;
+ case 7:
+ if ($ftype == TType::I64) {
+ $xfer += $input->readI64($this->txnId);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 8:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->clonePart);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
default:
$xfer += $input->skip($ftype);
break;
@@ -232,6 +270,16 @@ class RenamePartitionRequest
$xfer += $output->writeString($this->validWriteIdList);
$xfer += $output->writeFieldEnd();
}
+ if ($this->txnId !== null) {
+ $xfer += $output->writeFieldBegin('txnId', TType::I64, 7);
+ $xfer += $output->writeI64($this->txnId);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->clonePart !== null) {
+ $xfer += $output->writeFieldBegin('clonePart', TType::BOOL, 8);
+ $xfer += $output->writeBool($this->clonePart);
+ $xfer += $output->writeFieldEnd();
+ }
$xfer += $output->writeFieldStop();
$xfer += $output->writeStructEnd();
return $xfer;
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index 3eb49d578d..be48a2022c 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@ -25460,17 +25460,21 @@ class RenamePartitionRequest(object):
- partVals
- newPart
- validWriteIdList
+ - txnId
+ - clonePart
"""
- def __init__(self, catName=None, dbName=None, tableName=None, partVals=None, newPart=None, validWriteIdList=None,):
+ def __init__(self, catName=None, dbName=None, tableName=None, partVals=None, newPart=None, validWriteIdList=None, txnId=None, clonePart=None,):
self.catName = catName
self.dbName = dbName
self.tableName = tableName
self.partVals = partVals
self.newPart = newPart
self.validWriteIdList = validWriteIdList
+ self.txnId = txnId
+ self.clonePart = clonePart
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
@@ -25517,6 +25521,16 @@ class RenamePartitionRequest(object):
self.validWriteIdList = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
else:
iprot.skip(ftype)
+ elif fid == 7:
+ if ftype == TType.I64:
+ self.txnId = iprot.readI64()
+ else:
+ iprot.skip(ftype)
+ elif fid == 8:
+ if ftype == TType.BOOL:
+ self.clonePart = iprot.readBool()
+ else:
+ iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
@@ -25554,6 +25568,14 @@ class RenamePartitionRequest(object):
oprot.writeFieldBegin('validWriteIdList', TType.STRING, 6)
oprot.writeString(self.validWriteIdList.encode('utf-8') if sys.version_info[0] == 2 else self.validWriteIdList)
oprot.writeFieldEnd()
+ if self.txnId is not None:
+ oprot.writeFieldBegin('txnId', TType.I64, 7)
+ oprot.writeI64(self.txnId)
+ oprot.writeFieldEnd()
+ if self.clonePart is not None:
+ oprot.writeFieldBegin('clonePart', TType.BOOL, 8)
+ oprot.writeBool(self.clonePart)
+ oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
@@ -31579,6 +31601,8 @@ RenamePartitionRequest.thrift_spec = (
(4, TType.LIST, 'partVals', (TType.STRING, 'UTF8', False), None, ), # 4
(5, TType.STRUCT, 'newPart', [Partition, None], None, ), # 5
(6, TType.STRING, 'validWriteIdList', 'UTF8', None, ), # 6
+ (7, TType.I64, 'txnId', None, None, ), # 7
+ (8, TType.BOOL, 'clonePart', None, None, ), # 8
)
all_structs.append(RenamePartitionResponse)
RenamePartitionResponse.thrift_spec = (
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
index f4c52a77ba..dc35d61f89 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -7068,6 +7068,8 @@ class RenamePartitionRequest
PARTVALS = 4
NEWPART = 5
VALIDWRITEIDLIST = 6
+ TXNID = 7
+ CLONEPART = 8
FIELDS = {
CATNAME => {:type => ::Thrift::Types::STRING, :name => 'catName', :optional => true},
@@ -7075,7 +7077,9 @@ class RenamePartitionRequest
TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
PARTVALS => {:type => ::Thrift::Types::LIST, :name => 'partVals', :element => {:type => ::Thrift::Types::STRING}},
NEWPART => {:type => ::Thrift::Types::STRUCT, :name => 'newPart', :class => ::Partition},
- VALIDWRITEIDLIST => {:type => ::Thrift::Types::STRING, :name => 'validWriteIdList', :optional => true}
+ VALIDWRITEIDLIST => {:type => ::Thrift::Types::STRING, :name => 'validWriteIdList', :optional => true},
+ TXNID => {:type => ::Thrift::Types::I64, :name => 'txnId', :optional => true},
+ CLONEPART => {:type => ::Thrift::Types::BOOL, :name => 'clonePart', :optional => true}
}
def struct_fields; FIELDS; end
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 15c39f930f..5e8a81f466 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -110,6 +110,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
public static final String MANUALLY_INITIATED_COMPACTION = "manual";
public static final String TRUNCATE_SKIP_DATA_DELETION = "truncateSkipDataDeletion";
+ public static final String RENAME_PARTITION_MAKE_COPY = "renamePartitionMakeCopy";
/**
* Capabilities of the current client. If this client talks to a MetaStore server in a manner
@@ -584,10 +585,12 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public void renamePartition(String catName, String dbname, String tableName, List<String> part_vals,
- Partition newPart, String validWriteIds) throws TException {
+ Partition newPart, String validWriteIds, long txnId, boolean makeCopy) throws TException {
RenamePartitionRequest req = new RenamePartitionRequest(dbname, tableName, part_vals, newPart);
req.setCatName(catName);
req.setValidWriteIdList(validWriteIds);
+ req.setTxnId(txnId);
+ req.setClonePart(makeCopy);
client.rename_partition_req(req);
}
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index 5189b8a9a5..3056a598fb 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -2425,9 +2425,15 @@ public interface IMetaStoreClient {
* @throws TException
* if error in communicating with metastore server
*/
+ default void renamePartition(String catName, String dbname, String tableName, List<String> part_vals,
+ Partition newPart, String validWriteIds)
+ throws TException {
+ renamePartition(catName, dbname, tableName, part_vals, newPart, validWriteIds, 0, false);
+ }
+
void renamePartition(String catName, String dbname, String tableName, List<String> part_vals,
- Partition newPart, String validWriteIds)
- throws InvalidOperationException, MetaException, TException;
+ Partition newPart, String validWriteIds, long txnId, boolean makeCopy)
+ throws TException;
/**
* Get schema for a table, excluding the partition columns.
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
index 3e1eb1c327..45fc196efe 100755
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
@@ -448,6 +448,21 @@ public class Warehouse {
return false;
}
+ public boolean copyDir(Path sourcePath, Path destPath, boolean needCmRecycle) throws MetaException {
+ try {
+ if (needCmRecycle) {
+ cm.recycle(sourcePath, RecycleType.COPY, true);
+ }
+ FileSystem srcFs = getFs(sourcePath);
+ FileSystem destFs = getFs(destPath);
+ // TODO: this operation can be expensive depending on the size of data.
+ return FileUtils.copy(srcFs, sourcePath, destFs, destPath, false, false, conf);
+ } catch (Exception ex) {
+ MetaStoreUtils.throwMetaException(ex);
+ }
+ return false;
+ }
+
void addToChangeManagement(Path file) throws MetaException {
try {
cm.recycle(file, RecycleType.COPY, true);
diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
index f046cbf948..c04d553691 100644
--- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
@@ -2109,7 +2109,9 @@ struct RenamePartitionRequest {
3: required string tableName,
4: required list<string> partVals,
5: required Partition newPart,
- 6: optional string validWriteIdList
+ 6: optional string validWriteIdList,
+ 7: optional i64 txnId, // txnId associated with the rename operation
+ 8: optional bool clonePart // non-blocking rename
}
struct RenamePartitionResponse {
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
index 6950f7f680..3577440c4e 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
@@ -45,6 +45,7 @@ import java.util.List;
import java.util.Optional;
import static org.apache.hadoop.hive.metastore.HMSHandler.isMustPurge;
+import static org.apache.hadoop.hive.metastore.HiveMetaStoreClient.RENAME_PARTITION_MAKE_COPY;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.throwMetaException;
@@ -113,33 +114,33 @@ public class AcidEventListener extends TransactionalMetaStoreEventListener {
long currentTxn = Optional.ofNullable(context).map(EnvironmentContext::getProperties)
.map(prop -> prop.get("txnId")).map(Long::parseLong)
.orElse(0L);
-
- long writeId = Optional.ofNullable(context).map(EnvironmentContext::getProperties)
- .map(prop -> prop.get("writeId")).map(Long::parseLong)
- .orElse(0L);
-
- try {
+
if (currentTxn > 0) {
- CompactionRequest rqst = new CompactionRequest(
+ long writeId = Optional.of(context).map(EnvironmentContext::getProperties)
+ .map(prop -> prop.get("writeId")).map(Long::parseLong)
+ .orElse(0L);
+
+ try {
+ CompactionRequest rqst = new CompactionRequest(
table.getDbName(), table.getTableName(), CompactionType.MAJOR);
- rqst.setRunas(TxnUtils.findUserToRunAs(table.getSd().getLocation(), table, conf));
- rqst.putToProperties("ifPurge", Boolean.toString(isMustPurge(context, table)));
-
- Iterator<Partition> partitionIterator = partitionEvent.getPartitionIterator();
- while (partitionIterator.hasNext()) {
- Partition p = partitionIterator.next();
-
- List<FieldSchema> partCols = partitionEvent.getTable().getPartitionKeys(); // partition columns
- List<String> partVals = p.getValues();
- rqst.setPartitionname(Warehouse.makePartName(partCols, partVals));
- rqst.putToProperties("location", p.getSd().getLocation());
-
- txnHandler.submitForCleanup(rqst, writeId, currentTxn);
+ rqst.setRunas(TxnUtils.findUserToRunAs(table.getSd().getLocation(), table, conf));
+ rqst.putToProperties("ifPurge", Boolean.toString(isMustPurge(context, table)));
+
+ Iterator<Partition> partitionIterator = partitionEvent.getPartitionIterator();
+ while (partitionIterator.hasNext()) {
+ Partition p = partitionIterator.next();
+
+ List<FieldSchema> partCols = partitionEvent.getTable().getPartitionKeys(); // partition columns
+ List<String> partVals = p.getValues();
+ rqst.setPartitionname(Warehouse.makePartName(partCols, partVals));
+ rqst.putToProperties("location", p.getSd().getLocation());
+
+ txnHandler.submitForCleanup(rqst, writeId, currentTxn);
+ }
+ } catch (InterruptedException | IOException e) {
+ throwMetaException(e);
}
}
- } catch ( InterruptedException | IOException e) {
- throwMetaException(e);
- }
}
}
}
@@ -174,8 +175,37 @@ public class AcidEventListener extends TransactionalMetaStoreEventListener {
txnHandler = getTxnHandler();
txnHandler.onRename(t.getCatName(), t.getDbName(), t.getTableName(), oldPartName,
t.getCatName(), t.getDbName(), t.getTableName(), newPartName);
+
+ EnvironmentContext context = partitionEvent.getEnvironmentContext();
+ Table table = partitionEvent.getTable();
+
+ boolean clonePart = Optional.ofNullable(context).map(EnvironmentContext::getProperties)
+ .map(prop -> prop.get(RENAME_PARTITION_MAKE_COPY)).map(Boolean::parseBoolean)
+ .orElse(false);
+
+ if (clonePart) {
+ long currentTxn = Optional.of(context).map(EnvironmentContext::getProperties)
+ .map(prop -> prop.get("txnId")).map(Long::parseLong)
+ .orElse(0L);
+
+ try {
+ if (currentTxn > 0) {
+ CompactionRequest rqst = new CompactionRequest(
+ table.getDbName(), table.getTableName(), CompactionType.MAJOR);
+ rqst.setRunas(TxnUtils.findUserToRunAs(table.getSd().getLocation(), table, conf));
+ rqst.setPartitionname(oldPartName);
+
+ rqst.putToProperties("location", oldPart.getSd().getLocation());
+ rqst.putToProperties("ifPurge", Boolean.toString(isMustPurge(context, table)));
+ txnHandler.submitForCleanup(rqst, partitionEvent.getWriteId(), currentTxn);
+ }
+ } catch ( InterruptedException | IOException e) {
+ throwMetaException(e);
+ }
+ }
}
}
+
@Override
public void onAlterDatabase(AlterDatabaseEvent dbEvent) throws MetaException {
Database oldDb = dbEvent.getOldDatabase();
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index e458ae207d..1f8365e314 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -111,6 +111,7 @@ import static org.apache.hadoop.hive.common.AcidConstants.SOFT_DELETE_TABLE;
import static org.apache.hadoop.hive.common.AcidConstants.DELTA_DIGITS;
import static org.apache.hadoop.hive.common.repl.ReplConst.REPL_TARGET_DATABASE_PROPERTY;
+import static org.apache.hadoop.hive.metastore.HiveMetaStoreClient.RENAME_PARTITION_MAKE_COPY;
import static org.apache.hadoop.hive.metastore.HiveMetaStoreClient.TRUNCATE_SKIP_DATA_DELETION;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS;
import static org.apache.hadoop.hive.metastore.ExceptionHandler.handleException;
@@ -3364,7 +3365,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
} else {
// For Acid tables we don't need to delete the old files, only write an empty baseDir.
// Compaction and cleaner will take care of the rest
- addTruncateBaseFile(location, writeId, DataFormat.TRUNCATED);
+ addTruncateBaseFile(location, writeId, conf, DataFormat.TRUNCATED);
}
}
}
@@ -3383,16 +3384,16 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
* Add an empty baseDir with a truncate metadatafile
* @param location partition or table directory
* @param writeId allocated writeId
- * @throws Exception
+ * @throws MetaException
*/
- private void addTruncateBaseFile(Path location, long writeId, DataFormat dataFormat)
+ static void addTruncateBaseFile(Path location, long writeId, Configuration conf, DataFormat dataFormat)
throws MetaException {
if (location == null)
return;
Path basePath = new Path(location, AcidConstants.baseDir(writeId));
try {
- FileSystem fs = location.getFileSystem(getConf());
+ FileSystem fs = location.getFileSystem(conf);
fs.mkdirs(basePath);
// We can not leave the folder empty, otherwise it will be skipped at some file listing in AcidUtils
// No need for a data file, a simple metadata is enough
@@ -5061,7 +5062,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
}
// ok even if the data is not deleted
} else if (TxnUtils.isTransactionalTable(tbl) && writeId > 0) {
- addTruncateBaseFile(partPath, writeId, DataFormat.DROPPED);
+ addTruncateBaseFile(partPath, writeId, conf, DataFormat.DROPPED);
}
if (!listeners.isEmpty()) {
@@ -5332,7 +5333,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
Path partPath = new Path(part.getSd().getLocation());
verifyIsWritablePath(partPath);
- addTruncateBaseFile(partPath, writeId, DataFormat.DROPPED);
+ addTruncateBaseFile(partPath, writeId, conf, DataFormat.DROPPED);
}
}
}
@@ -5822,10 +5823,13 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
}
@Override
- public RenamePartitionResponse rename_partition_req(
- RenamePartitionRequest req) throws InvalidOperationException ,MetaException ,TException {
+ public RenamePartitionResponse rename_partition_req(RenamePartitionRequest req) throws TException {
+ EnvironmentContext context = new EnvironmentContext();
+ context.putToProperties(RENAME_PARTITION_MAKE_COPY, String.valueOf(req.isClonePart()));
+ context.putToProperties("txnId", String.valueOf(req.getTxnId()));
+
rename_partition(req.getCatName(), req.getDbName(), req.getTableName(), req.getPartVals(),
- req.getNewPart(), null, req.getValidWriteIdList());
+ req.getNewPart(), context, req.getValidWriteIdList());
return new RenamePartitionResponse();
};
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index aa32d60eb7..66d62f0530 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -24,6 +24,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.common.AcidMetaDataFile.DataFormat;
import org.apache.hadoop.hive.common.repl.ReplConst;
import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
@@ -66,9 +67,12 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.stream.Collectors;
import java.util.LinkedList;
+import java.util.Optional;
+import static org.apache.hadoop.hive.metastore.HMSHandler.addTruncateBaseFile;
import static org.apache.hadoop.hive.metastore.HiveMetaHook.ALTERLOCATION;
import static org.apache.hadoop.hive.metastore.HiveMetaHook.ALTER_TABLE_OPERATION_TYPE;
+import static org.apache.hadoop.hive.metastore.HiveMetaStoreClient.RENAME_PARTITION_MAKE_COPY;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
@@ -673,9 +677,26 @@ public class HiveAlterHandler implements AlterHandler {
if (!wh.mkdirs(destParentPath)) {
throw new MetaException("Unable to create path " + destParentPath);
}
-
- //rename the data directory
- wh.renameDir(srcPath, destPath, ReplChangeManager.shouldEnableCm(db, tbl));
+
+ boolean clonePart = Optional.ofNullable(environmentContext)
+ .map(EnvironmentContext::getProperties)
+ .map(prop -> prop.get(RENAME_PARTITION_MAKE_COPY))
+ .map(Boolean::parseBoolean)
+ .orElse(false);
+ long writeId = new_part.getWriteId();
+
+ if (writeId > 0 && clonePart) {
+ LOG.debug("Making a copy of the partition directory: {} under a new location: {}", srcPath, destPath);
+
+ if (!wh.copyDir(srcPath, destPath, ReplChangeManager.shouldEnableCm(db, tbl))) {
+ LOG.error("Copy failed for source: " + srcPath + " to destination: " + destPath);
+ throw new IOException("File copy failed.");
+ }
+ addTruncateBaseFile(srcPath, writeId, conf, DataFormat.DROPPED);
+ } else {
+ //rename the data directory
+ wh.renameDir(srcPath, destPath, ReplChangeManager.shouldEnableCm(db, tbl));
+ }
LOG.info("Partition directory rename from " + srcPath + " to " + destPath + " done.");
dataWasMoved = true;
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
index 5a60e99cc0..5bf7b0d8a8 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
@@ -3611,8 +3611,8 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
@Override
public void renamePartition(String catName, String dbname, String tableName,
- List<String> part_vals, Partition newPart, String validWriteIds)
- throws InvalidOperationException, MetaException, TException {
+ List<String> part_vals, Partition newPart, String validWriteIds, long txnId, boolean makeCopy)
+ throws TException {
throw new UnsupportedOperationException();
}