You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2019/10/02 10:42:03 UTC
[hive] branch master updated: HIVE-22137: Implement alter/rename
partition related methods on temporary tables (Laszlo Pinter via Peter
Vary)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 1ed457c HIVE-22137: Implement alter/rename partition related methods on temporary tables (Laszlo Pinter via Peter Vary)
1ed457c is described below
commit 1ed457c24b81f1c72df7071dc7835dc81a393651
Author: Laszlo Pinter <lp...@cloudera.com>
AuthorDate: Wed Oct 2 11:37:43 2019 +0200
HIVE-22137: Implement alter/rename partition related methods on temporary tables (Laszlo Pinter via Peter Vary)
---
.../hadoop/hive/ql/metadata/PartitionTree.java | 93 +++++++-
.../ql/metadata/SessionHiveMetaStoreClient.java | 43 ++++
.../apache/hadoop/hive/ql/metadata/TempTable.java | 23 +-
...iveMetastoreClientAlterPartitionsTempTable.java | 249 +++++++++++++++++++++
.../hadoop/hive/metastore/IMetaStoreClient.java | 4 +-
.../hive/metastore/client/TestAlterPartitions.java | 86 ++++---
6 files changed, 455 insertions(+), 43 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/PartitionTree.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/PartitionTree.java
index c84c3ef..00b4301 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/PartitionTree.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/PartitionTree.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.metadata;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -37,7 +38,7 @@ import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.makePartName
* via references.
*/
final class PartitionTree {
- private final Map<String, org.apache.hadoop.hive.metastore.api.Partition> parts = new LinkedHashMap<>();
+ private Map<String, org.apache.hadoop.hive.metastore.api.Partition> parts = new LinkedHashMap<>();
private final org.apache.hadoop.hive.metastore.api.Table tTable;
PartitionTree(org.apache.hadoop.hive.metastore.api.Table t) {
@@ -67,7 +68,7 @@ final class PartitionTree {
* @param partVals partition values for this partition, must be in the same order as the
* partition keys of the table.
* @return the partition object, or if not found null.
- * @throws MetaException
+ * @throws MetaException partition values are incorrect.
*/
Partition getPartition(List<String> partVals) throws MetaException {
String partName = makePartName(tTable.getPartitionKeys(), partVals);
@@ -80,7 +81,8 @@ final class PartitionTree {
* @param partitions The partitions to add
* @param ifNotExists only add partitions if they don't exist
* @return the partitions that were added
- * @throws MetaException
+ * @throws MetaException partition metadata is incorrect
+ * @throws AlreadyExistsException if the partition with the same name already exists.
*/
List<Partition> addPartitions(List<Partition> partitions, boolean ifNotExists)
throws MetaException, AlreadyExistsException {
@@ -138,7 +140,7 @@ final class PartitionTree {
* Remove a partition from the table.
* @param partVals partition values, must be not null
* @return the instance of the dropped partition, if the remove was successful, otherwise false
- * @throws MetaException
+ * @throws MetaException partition with the provided partition values cannot be found.
*/
Partition dropPartition(List<String> partVals) throws MetaException, NoSuchObjectException {
String partName = makePartName(tTable.getPartitionKeys(), partVals);
@@ -148,4 +150,87 @@ final class PartitionTree {
}
return parts.remove(partName);
}
+
+ /**
+ * Alter an existing partition. The flow is following:
+ * <p>
+ * 1) search for existing partition
+ * 2) if found delete it
+ * 3) insert new partition
+ * </p>
+ * @param oldPartitionVals the values of existing partition, which is altered, must be not null.
+ * @param newPartition the new partition, must be not null.
+ * @param isRename true, if rename is requested, meaning that all properties of partition can be changed, except
+ * of its location.
+ * @throws MetaException table or db name is altered.
+ * @throws InvalidOperationException the new partition values are null, or the old partition cannot be found.
+ */
+ void alterPartition(List<String> oldPartitionVals, Partition newPartition, boolean isRename)
+ throws MetaException, InvalidOperationException, NoSuchObjectException {
+ if (oldPartitionVals == null || oldPartitionVals.isEmpty()) {
+ throw new InvalidOperationException("Old partition values cannot be null or empty.");
+ }
+ if (newPartition == null) {
+ throw new InvalidOperationException("New partition cannot be null.");
+ }
+ Partition oldPartition = getPartition(oldPartitionVals);
+ if (oldPartition == null) {
+ throw new InvalidOperationException(
+ "Partition with partition values " + Arrays.toString(oldPartitionVals.toArray()) + " is not found.");
+ }
+ if (!oldPartition.getDbName().equals(newPartition.getDbName())) {
+ throw new MetaException("Db name cannot be altered.");
+ }
+ if (!oldPartition.getTableName().equals(newPartition.getTableName())) {
+ throw new MetaException("Table name cannot be altered.");
+ }
+ if (isRename) {
+ newPartition.getSd().setLocation(oldPartition.getSd().getLocation());
+ }
+ dropPartition(oldPartitionVals);
+ String partName = makePartName(tTable.getPartitionKeys(), newPartition.getValues());
+ if (parts.containsKey(partName)) {
+ throw new InvalidOperationException("Partition " + partName + " already exists");
+ }
+ parts.put(partName, newPartition);
+ }
+
+ /**
+ * Alter multiple partitions. This operation is transactional.
+ * @param newParts list of new partitions, must be not null.
+ * @throws MetaException table or db name is altered.
+ * @throws InvalidOperationException the new partition values are null, or the old partition cannot be found.
+ * @throws NoSuchObjectException the old partition cannot be found.
+ */
+ void alterPartitions(List<Partition> newParts)
+ throws MetaException, InvalidOperationException, NoSuchObjectException {
+ //altering partitions in a batch must be transactional, therefore bofore starting the altering, clone the original
+ //partitions map. If something fails, revert it back.
+ Map<String, Partition> clonedPartitions = new LinkedHashMap<>();
+ parts.forEach((key, value) -> clonedPartitions.put(key, new Partition(value)));
+ for (Partition partition : newParts) {
+ try {
+ if (partition == null) {
+ throw new InvalidOperationException("New partition cannot be null.");
+ }
+ alterPartition(partition.getValues(), partition, false);
+ } catch (MetaException | InvalidOperationException | NoSuchObjectException e) {
+ parts = clonedPartitions;
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Rename an existing partition.
+ * @param oldPartitionVals the values of existing partition, which is renamed, must be not null.
+ * @param newPart the new partition, must be not null.
+ * @throws MetaException table or db name is altered.
+ * @throws InvalidOperationException the new partition values are null, or the old partition cannot be altered.
+ * @throws NoSuchObjectException the old partition cannot be found.
+ */
+ void renamePartition(List<String> oldPartitionVals, Partition newPart)
+ throws MetaException, InvalidOperationException, NoSuchObjectException {
+ alterPartition(oldPartitionVals, newPart, true);
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index a5b16d1..2467ee3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -1341,6 +1341,49 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
throw new MetaException("Exchanging partitions between temporary and non-temporary tables is not supported.");
}
+ @Override
+ public void alter_partition(String catName, String dbName, String tblName, Partition newPart,
+ EnvironmentContext environmentContext) throws TException {
+ alter_partition(catName, dbName, tblName, newPart, environmentContext, null);
+ }
+
+ @Override
+ public void alter_partition(String catName, String dbName, String tblName, Partition newPart,
+ EnvironmentContext environmentContext, String writeIdList)
+ throws TException {
+ org.apache.hadoop.hive.metastore.api.Table table = getTempTable(dbName, tblName);
+ if (table == null) {
+ super.alter_partition(catName, dbName, tblName, newPart, environmentContext, writeIdList);
+ return;
+ }
+ TempTable tt = getPartitionedTempTable(table);
+ tt.alterPartition(newPart);
+ }
+
+ @Override
+ public void alter_partitions(String catName, String dbName, String tblName, List<Partition> newParts,
+ EnvironmentContext environmentContext, String writeIdList, long writeId) throws TException {
+ org.apache.hadoop.hive.metastore.api.Table table = getTempTable(dbName, tblName);
+ if (table == null) {
+ super.alter_partitions(catName, dbName, tblName, newParts, environmentContext, writeIdList, writeId);
+ return;
+ }
+ TempTable tt = getPartitionedTempTable(table);
+ tt.alterPartitions(newParts);
+ }
+
+ @Override
+ public void renamePartition(String catName, String dbname, String tableName, List<String> partitionVals,
+ Partition newPart, String validWriteIds) throws TException {
+ org.apache.hadoop.hive.metastore.api.Table table = getTempTable(dbname, tableName);
+ if (table == null) {
+ super.renamePartition(catName, dbname, tableName, partitionVals, newPart, validWriteIds);
+ return;
+ }
+ TempTable tt = getPartitionedTempTable(table);
+ tt.renamePartition(partitionVals, newPart);
+ }
+
private List<Partition> exchangePartitions(Map<String, String> partitionSpecs,
org.apache.hadoop.hive.metastore.api.Table sourceTable, TempTable sourceTempTable,
org.apache.hadoop.hive.metastore.api.Table destTable, TempTable destTempTable) throws TException {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/TempTable.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/TempTable.java
index fa6dddc..e659b54 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/TempTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/TempTable.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.metadata;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -103,11 +104,11 @@ public final class TempTable {
return checkPrivilegesForPartition(partition, userName, groupNames) ? partition : null;
}
- List<Partition> listPartitions() throws MetaException {
+ List<Partition> listPartitions() {
return pTree.listPartitions();
}
- List<Partition> listPartitionsWithAuthInfo(String userName, List<String> groupNames) throws MetaException {
+ List<Partition> listPartitionsWithAuthInfo(String userName, List<String> groupNames) {
List<Partition> partitions = listPartitions();
List<Partition> result = new ArrayList<>();
partitions.forEach(p -> {
@@ -130,7 +131,7 @@ public final class TempTable {
return result;
}
- boolean checkPrivilegesForPartition(Partition partition, String userName, List<String> groupNames) {
+ private boolean checkPrivilegesForPartition(Partition partition, String userName, List<String> groupNames) {
if ((userName == null || userName.isEmpty()) && (groupNames == null || groupNames.isEmpty())) {
return true;
}
@@ -162,7 +163,7 @@ public final class TempTable {
Partition dropPartition(String partitionName) throws MetaException, NoSuchObjectException {
Map<String, String> specFromName = makeSpecFromName(partitionName);
- if (specFromName == null || specFromName.isEmpty()) {
+ if (specFromName.isEmpty()) {
throw new NoSuchObjectException("Invalid partition name " + partitionName);
}
List<String> pVals = new ArrayList<>();
@@ -177,4 +178,18 @@ public final class TempTable {
return pTree.dropPartition(pVals);
}
+ void alterPartition(Partition partition) throws MetaException, InvalidOperationException, NoSuchObjectException {
+ pTree.alterPartition(partition.getValues(), partition, false);
+ }
+
+ void alterPartitions(List<Partition> newParts)
+ throws MetaException, InvalidOperationException, NoSuchObjectException {
+ pTree.alterPartitions(newParts);
+ }
+
+ void renamePartition(List<String> partitionVals, Partition newPart)
+ throws MetaException, InvalidOperationException, NoSuchObjectException {
+ pTree.renamePartition(partitionVals, newPart);
+ }
+
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAlterPartitionsTempTable.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAlterPartitionsTempTable.java
new file mode 100644
index 0000000..72977c7
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAlterPartitionsTempTable.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.metadata;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.client.CustomIgnoreRule;
+import org.apache.hadoop.hive.metastore.client.TestAlterPartitions;
+import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.thrift.TException;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+/**
+ * Test class for alter/rename partitions related methods on temporary tables.
+ */
+@RunWith(Parameterized.class)
+@Category(MetastoreCheckinTest.class)
+public class TestSessionHiveMetastoreClientAlterPartitionsTempTable extends TestAlterPartitions {
+
+ private HiveConf conf;
+
+ private static final String PART_PRIV = "PARTITION_LEVEL_PRIVILEGE";
+
+ public TestSessionHiveMetastoreClientAlterPartitionsTempTable(String name, AbstractMetaStoreService metaStore) {
+ super(name, metaStore);
+ ignoreRule = new CustomIgnoreRule();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ initHiveConf();
+ SessionState.start(conf);
+ setClient(Hive.get(conf).getMSC());
+ cleanDB();
+ createDB(DB_NAME);
+ }
+
+ private void initHiveConf() throws HiveException {
+ conf = Hive.get().getConf();
+ conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true);
+ }
+
+ @Override
+ protected Table createTestTable(IMetaStoreClient client, String dbName, String tableName,
+ List<String> partCols, boolean setPartitionLevelPrivilages)
+ throws Exception {
+ TableBuilder builder = new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tableName)
+ .addCol("id", "int")
+ .addCol("name", "string")
+ .setTemporary(true);
+
+ partCols.forEach(col -> builder.addPartCol(col, "string"));
+ Table table = builder.build(getMetaStore().getConf());
+
+ if (setPartitionLevelPrivilages) {
+ table.putToParameters(PART_PRIV, "true");
+ }
+
+ client.createTable(table);
+ return table;
+ }
+
+ @Override
+ protected void addPartition(IMetaStoreClient client, Table table, List<String> values)
+ throws TException {
+ PartitionBuilder builder = new PartitionBuilder().inTable(table);
+ values.forEach(builder::addValue);
+ Partition partition = builder.build(conf);
+ getClient().add_partition(partition);
+ }
+
+ @Override
+ protected void addPartitions(IMetaStoreClient client, Table table, List<String> values) throws Exception {
+ List<Partition> partitions = new ArrayList<>();
+ for (int i = 0; i < values.size(); i++) {
+ partitions.add(new PartitionBuilder().inTable(table)
+ .addValue(values.get(i))
+ .setLocation(MetaStoreTestUtils.getTestWarehouseDir(values.get(i) + i))
+ .build(conf));
+ }
+ client.add_partitions(partitions);
+ }
+
+ @Override
+ protected void assertPartitionUnchanged(Partition partition, List<String> testValues,
+ List<String> partCols) throws Exception {
+ assertFalse(partition.getParameters().containsKey("hmsTestParam001"));
+
+ List<String> expectedKVPairs = new ArrayList<>();
+ for (int i = 0; i < partCols.size(); ++i) {
+ expectedKVPairs.add(partCols.get(i) + "=" + testValues.get(i));
+ }
+ Table table = getClient().getTable(partition.getDbName(), partition.getTableName());
+ String partPath = String.join("/", expectedKVPairs);
+ assertEquals(partition.getSd().getLocation(), table.getSd().getLocation() + "/" + partPath);
+ assertEquals(2, partition.getSd().getCols().size());
+ }
+
+ @Override
+ protected void assertPartitionChanged(Partition partition, List<String> testValues,
+ List<String> partCols) throws Exception {
+ assertEquals("testValue001", partition.getParameters().get("hmsTestParam001"));
+
+ List<String> expectedKVPairs = new ArrayList<>();
+ for (int i = 0; i < partCols.size(); ++i) {
+ expectedKVPairs.add(partCols.get(i) + "=" + testValues.get(i));
+ }
+ Table table = getClient().getTable(partition.getDbName(), partition.getTableName());
+ String partPath = String.join("/", expectedKVPairs);
+ assertEquals(partition.getSd().getLocation(), table.getSd().getLocation() + "/" + partPath + "/hh=01");
+ assertEquals(NEW_CREATE_TIME, partition.getCreateTime());
+ assertEquals(NEW_CREATE_TIME, partition.getLastAccessTime());
+ assertEquals(3, partition.getSd().getCols().size());
+ }
+
+ @Override
+ @Test(expected = InvalidOperationException.class)
+ public void testRenamePartitionNullNewPart() throws Exception {
+ super.testRenamePartitionNullNewPart();
+ }
+
+ @Override
+ @Test(expected = InvalidOperationException.class)
+ public void testAlterPartitionsNullPartition() throws Exception {
+ super.testAlterPartitionsNullPartition();
+ }
+
+ @Override
+ @Test(expected = InvalidOperationException.class)
+ public void testAlterPartitionsWithEnvironmentCtxNullPartition() throws Exception {
+ super.testAlterPartitionsWithEnvironmentCtxNullPartition();
+ }
+
+ @Override
+ @Test(expected = InvalidOperationException.class)
+ public void testAlterPartitionsNullPartitions() throws Exception {
+ super.testAlterPartitionsNullPartitions();
+ }
+
+ @Override
+ @Test(expected = InvalidOperationException.class)
+ public void testAlterPartitionsWithEnvironmentCtxNullPartitions() throws Exception {
+ super.testAlterPartitionsWithEnvironmentCtxNullPartitions();
+ }
+
+ @Test
+ public void testAlterPartitionsCheckRollbackNullPartition() throws Exception {
+ createTable4PartColsParts(getClient());
+ List<Partition> oldParts = getClient().listPartitions(DB_NAME, TABLE_NAME, (short) -1);
+ assertPartitionRollback(oldParts, Lists.newArrayList(oldParts.get(0), null, oldParts.get(1)));
+ }
+
+ @Test
+ public void testAlterPartitionsCheckRollbackNullPartitions() throws Exception {
+ createTable4PartColsParts(getClient());
+ assertPartitionRollback(getClient().listPartitions(DB_NAME, TABLE_NAME, (short) -1),
+ Lists.newArrayList(null, null));
+ }
+
+ @Test
+ public void testAlterPartitionsCheckRollbackPartValsNull() throws Exception {
+ createTable4PartColsParts(getClient());
+ List<Partition> oldParts = getClient().listPartitions(DB_NAME, TABLE_NAME, (short) -1);
+ Partition partition = new Partition(oldParts.get(0));
+ partition.setValues(null);
+ assertPartitionRollback(oldParts, Lists.newArrayList(partition));
+ }
+
+ @Test
+ public void testAlterPartitionsCheckRollbackUnknownPartition() throws Exception {
+ createTable4PartColsParts(getClient());
+ Table table = getClient().getTable(DB_NAME, TABLE_NAME);
+ Partition newPart1 =
+ new PartitionBuilder().inTable(table).addValue("1111").addValue("1111").addValue("11").build(conf);
+ List<Partition> oldPartitions = getClient().listPartitions(DB_NAME, TABLE_NAME, (short) -1);
+ Partition newPart2 = new Partition(oldPartitions.get(0));
+ makeTestChangesOnPartition(newPart2);
+ assertPartitionRollback(oldPartitions, Lists.newArrayList(newPart2, newPart1));
+ }
+
+ @Test
+ public void testAlterPartitionsCheckRollbackChangeDBName() throws Exception {
+ createTable4PartColsParts(getClient());
+ List<Partition> oldPartitions = getClient().listPartitions(DB_NAME, TABLE_NAME, (short) -1);
+ Partition newPart1 = new Partition(oldPartitions.get(3));
+ newPart1.setDbName(DB_NAME + "_changed");
+ assertPartitionRollback(oldPartitions, Lists.newArrayList(oldPartitions.get(0), oldPartitions.get(1), newPart1,
+ oldPartitions.get(2)));
+ }
+
+ @Test
+ public void testAlterPartitionsCheckRollbackChangeTableName() throws Exception {
+ createTable4PartColsParts(getClient());
+ List<Partition> oldPartitions = getClient().listPartitions(DB_NAME, TABLE_NAME, (short) -1);
+ Partition newPart1 = new Partition(oldPartitions.get(3));
+ newPart1.setTableName(TABLE_NAME + "_changed");
+ assertPartitionRollback(oldPartitions, Lists.newArrayList(oldPartitions.get(0), oldPartitions.get(1), newPart1,
+ oldPartitions.get(2)));
+ }
+
+ @SuppressWarnings("deprecation")
+ private void assertPartitionRollback(List<Partition> oldParts, List<Partition> alterParts) throws TException {
+ try {
+ getClient().alter_partitions(DB_NAME, TABLE_NAME, alterParts);
+ } catch (MetaException | InvalidOperationException e) {
+ assertEquals(oldParts, getClient().listPartitions(DB_NAME, TABLE_NAME, (short) -1));
+ return;
+ }
+ fail("Exception should have been thrown.");
+ }
+
+}
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index 6c7d80e..109b7e1 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -2230,7 +2230,7 @@ public interface IMetaStoreClient {
* @param newPart
* new partition
* @throws InvalidOperationException
- * if srcFs and destFs are different
+ * if srcFs and destFs are different, or trying to rename to an already existing partition name
* @throws MetaException
* if error in updating metadata
* @throws TException
@@ -2253,7 +2253,7 @@ public interface IMetaStoreClient {
* @param newPart
* new partition
* @throws InvalidOperationException
- * if srcFs and destFs are different
+ * if srcFs and destFs are different, or trying to rename to an already existing partition name
* @throws MetaException
* if error in updating metadata
* @throws TException
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java
index 4fc3688..4de673e 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.metastore.client;
-import java.net.ProtocolException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -48,7 +47,6 @@ import org.apache.thrift.transport.TTransportException;
import com.google.common.collect.Lists;
import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -70,12 +68,12 @@ import static org.junit.Assert.fail;
@RunWith(Parameterized.class)
@Category(MetastoreCheckinTest.class)
public class TestAlterPartitions extends MetaStoreClientTest {
- private static final int NEW_CREATE_TIME = 123456789;
+ protected static final int NEW_CREATE_TIME = 123456789;
private AbstractMetaStoreService metaStore;
private IMetaStoreClient client;
- private static final String DB_NAME = "testpartdb";
- private static final String TABLE_NAME = "testparttable";
+ protected static final String DB_NAME = "testpartdb";
+ protected static final String TABLE_NAME = "testparttable";
private static final List<String> PARTCOL_SCHEMA = Lists.newArrayList("yyyy", "mm", "dd");
public TestAlterPartitions(String name, AbstractMetaStoreService metaStore) {
@@ -88,8 +86,7 @@ public class TestAlterPartitions extends MetaStoreClientTest {
client = metaStore.getClient();
// Clean up the database
- client.dropDatabase(DB_NAME, true, true, true);
- metaStore.cleanWarehouseDirs();
+ cleanDB();
createDB(DB_NAME);
}
@@ -108,13 +105,34 @@ public class TestAlterPartitions extends MetaStoreClientTest {
}
}
- private void createDB(String dbName) throws TException {
+ public AbstractMetaStoreService getMetaStore() {
+ return metaStore;
+ }
+
+ public void setMetaStore(AbstractMetaStoreService metaStore) {
+ this.metaStore = metaStore;
+ }
+
+ protected IMetaStoreClient getClient() {
+ return client;
+ }
+
+ protected void setClient(IMetaStoreClient client) {
+ this.client = client;
+ }
+
+ protected void cleanDB() throws Exception{
+ client.dropDatabase(DB_NAME, true, true, true);
+ metaStore.cleanWarehouseDirs();
+ }
+
+ protected void createDB(String dbName) throws TException {
new DatabaseBuilder().
setName(dbName).
create(client, metaStore.getConf());
}
- private Table createTestTable(IMetaStoreClient client, String dbName, String tableName,
+ protected Table createTestTable(IMetaStoreClient client, String dbName, String tableName,
List<String> partCols, boolean setPartitionLevelPrivilages)
throws Exception {
TableBuilder builder = new TableBuilder()
@@ -134,14 +152,25 @@ public class TestAlterPartitions extends MetaStoreClientTest {
return table;
}
- private void addPartition(IMetaStoreClient client, Table table, List<String> values)
+ protected void addPartition(IMetaStoreClient client, Table table, List<String> values)
throws TException {
PartitionBuilder partitionBuilder = new PartitionBuilder().inTable(table);
values.forEach(val -> partitionBuilder.addValue(val));
client.add_partition(partitionBuilder.build(metaStore.getConf()));
}
- private List<List<String>> createTable4PartColsParts(IMetaStoreClient client) throws
+ protected void addPartitions(IMetaStoreClient client, Table table, List<String> values) throws Exception {
+ List<Partition> partitions = new ArrayList<>();
+ for (int i = 0; i < values.size(); i++) {
+ partitions.add(new PartitionBuilder().inTable(table)
+ .addValue(values.get(i))
+ .setLocation(MetaStoreTestUtils.getTestWarehouseDir(values.get(i) + i))
+ .build(metaStore.getConf()));
+ }
+ client.add_partitions(partitions);
+ }
+
+ protected List<List<String>> createTable4PartColsParts(IMetaStoreClient client) throws
Exception {
Table t = createTestTable(client, DB_NAME, TABLE_NAME, PARTCOL_SCHEMA, false);
List<List<String>> testValues = Lists.newArrayList(
@@ -165,7 +194,7 @@ public class TestAlterPartitions extends MetaStoreClientTest {
}
}
- private static void makeTestChangesOnPartition(Partition partition) {
+ protected static void makeTestChangesOnPartition(Partition partition) {
partition.getParameters().put("hmsTestParam001", "testValue001");
partition.setCreateTime(NEW_CREATE_TIME);
partition.setLastAccessTime(NEW_CREATE_TIME);
@@ -173,8 +202,8 @@ public class TestAlterPartitions extends MetaStoreClientTest {
partition.getSd().getCols().add(new FieldSchema("newcol", "string", ""));
}
- private void assertPartitionUnchanged(Partition partition, List<String> testValues,
- List<String> partCols) throws MetaException {
+ protected void assertPartitionUnchanged(Partition partition, List<String> testValues,
+ List<String> partCols) throws Exception {
assertFalse(partition.getParameters().containsKey("hmsTestParam001"));
List<String> expectedKVPairs = new ArrayList<>();
@@ -189,8 +218,8 @@ public class TestAlterPartitions extends MetaStoreClientTest {
assertEquals(2, partition.getSd().getCols().size());
}
- private void assertPartitionChanged(Partition partition, List<String> testValues,
- List<String> partCols) throws MetaException {
+ protected void assertPartitionChanged(Partition partition, List<String> testValues,
+ List<String> partCols) throws Exception {
assertEquals("testValue001", partition.getParameters().get("hmsTestParam001"));
List<String> expectedKVPairs = new ArrayList<>();
@@ -230,6 +259,7 @@ public class TestAlterPartitions extends MetaStoreClientTest {
}
@Test
+ @ConditionalIgnoreOnSessionHiveMetastoreClient
public void otherCatalog() throws TException {
String catName = "alter_partition_catalog";
Catalog cat = new CatalogBuilder()
@@ -307,24 +337,10 @@ public class TestAlterPartitions extends MetaStoreClientTest {
@SuppressWarnings("deprecation")
@Test
- public void deprecatedCalls() throws TException {
+ public void deprecatedCalls() throws Exception {
String tableName = "deprecated_table";
- Table table = new TableBuilder()
- .setTableName(tableName)
- .addCol("id", "int")
- .addCol("name", "string")
- .addPartCol("partcol", "string")
- .create(client, metaStore.getConf());
-
- Partition[] parts = new Partition[5];
- for (int i = 0; i < 5; i++) {
- parts[i] = new PartitionBuilder()
- .inTable(table)
- .addValue("a" + i)
- .setLocation(MetaStoreTestUtils.getTestWarehouseDir("a" + i))
- .build(metaStore.getConf());
- }
- client.add_partitions(Arrays.asList(parts));
+ Table table = createTestTable(getClient(), DEFAULT_DATABASE_NAME, tableName, Arrays.asList("partcol"), false);
+ addPartitions(getClient(), table, Arrays.asList("a0", "a1", "a2", "a3", "a4"));
Partition newPart =
client.getPartition(DEFAULT_DATABASE_NAME, tableName, Collections.singletonList("a0"));
@@ -397,6 +413,7 @@ public class TestAlterPartitions extends MetaStoreClientTest {
}
@Test(expected = InvalidOperationException.class)
+ @ConditionalIgnoreOnSessionHiveMetastoreClient
public void testAlterPartitionBogusCatalogName() throws Exception {
createTable4PartColsParts(client);
List<Partition> partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1);
@@ -683,6 +700,7 @@ public class TestAlterPartitions extends MetaStoreClientTest {
}
@Test(expected = InvalidOperationException.class)
+ @ConditionalIgnoreOnSessionHiveMetastoreClient
public void testAlterPartitionsBogusCatalogName() throws Exception {
createTable4PartColsParts(client);
Partition part = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0);
@@ -859,6 +877,7 @@ public class TestAlterPartitions extends MetaStoreClientTest {
}
@Test(expected = InvalidOperationException.class)
+ @ConditionalIgnoreOnSessionHiveMetastoreClient
public void testAlterPartitionsWithEnvironmentCtxBogusCatalogName() throws Exception {
createTable4PartColsParts(client);
Partition part = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0);
@@ -1078,6 +1097,7 @@ public class TestAlterPartitions extends MetaStoreClientTest {
}
@Test(expected = InvalidOperationException.class)
+ @ConditionalIgnoreOnSessionHiveMetastoreClient
public void testRenamePartitionBogusCatalogName() throws Exception {
List<List<String>> oldValues = createTable4PartColsParts(client);
List<Partition> oldParts = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1);