You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2019/09/24 16:08:41 UTC
[hive] branch master updated: HIVE-22084: Implement exchange
partitions related methods on temporary tables (Laszlo Pintet via Peter
Vary)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 48ae7ef HIVE-22084: Implement exchange partitions related methods on temporary tables (Laszlo Pintet via Peter Vary)
48ae7ef is described below
commit 48ae7ef8d46c29918364a3af8f5f2fba3e92fd30
Author: Laszlo Pinter <lp...@cloudera.com>
AuthorDate: Tue Sep 24 17:33:53 2019 +0200
HIVE-22084: Implement exchange partitions related methods on temporary tables (Laszlo Pintet via Peter Vary)
---
.../ql/metadata/SessionHiveMetaStoreClient.java | 120 ++++++-
...MetastoreClientExchangePartitionsTempTable.java | 361 +++++++++++++++++++++
.../metastore/client/TestExchangePartitions.java | 70 ++--
3 files changed, 519 insertions(+), 32 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index 506bf5d..acd6f72 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -71,7 +71,6 @@ import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownTableException;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.shims.HadoopShims;
@@ -85,7 +84,13 @@ import static org.apache.hadoop.hive.metastore.Warehouse.getCatalogQualifiedTabl
import static org.apache.hadoop.hive.metastore.Warehouse.makePartName;
import static org.apache.hadoop.hive.metastore.Warehouse.makeSpecFromName;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.compareFieldColumns;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getColumnNamesForTable;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getPvals;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.isExternalTable;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.makePartNameMatcher;
+
/**
* todo: This need review re: thread safety. Various places (see callsers of
@@ -199,7 +204,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
return deepCopy(table); // Original method used deepCopy(), do the same here.
}
// Try underlying client
- return super.getTable(MetaStoreUtils.getDefaultCatalog(conf), dbname, name, getColStats, engine);
+ return super.getTable(getDefaultCatalog(conf), dbname, name, getColStats, engine);
}
// Need to override this one too or dropTable breaks because it doesn't find the table when checks
@@ -574,8 +579,8 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
// Add temp table info to current session
Table tTable = new Table(tbl);
if (!isVirtualTable) {
- StatsSetupConst.setStatsStateForCreateTable(tbl.getParameters(),
- org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getColumnNamesForTable(tbl), StatsSetupConst.TRUE);
+ StatsSetupConst.setStatsStateForCreateTable(tbl.getParameters(), getColumnNamesForTable(tbl),
+ StatsSetupConst.TRUE);
}
if (tables == null) {
tables = new HashMap<String, Table>();
@@ -782,7 +787,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
removePartitionedTempTable(table);
// Delete table data
- if (deleteData && !MetaStoreUtils.isExternalTable(table)) {
+ if (deleteData && !isExternalTable(table)) {
try {
boolean ifPurge = false;
if (envContext != null){
@@ -1149,7 +1154,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
if (partialPartVals == null || partialPartVals.isEmpty()) {
throw new MetaException("Partition partial vals cannot be null or empty");
}
- String partNameMatcher = MetaStoreUtils.makePartNameMatcher(tTable, partialPartVals, ".*");
+ String partNameMatcher = makePartNameMatcher(tTable, partialPartVals, ".*");
List<Partition> matchedPartitions = new ArrayList<>();
for (String key : parts.keySet()) {
if (key.matches(partNameMatcher)) {
@@ -1206,7 +1211,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
}
TempTable tt = getPartitionedTempTable(table);
checkPartitionProperties(partition);
- Path partitionLocation = getPartitionLocation(table, partition);
+ Path partitionLocation = getPartitionLocation(table, partition, false);
Partition result = tt.addPartition(deepCopy(partition));
createAndSetLocationForAddedPartition(result, partitionLocation);
return result;
@@ -1566,6 +1571,97 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
+ "supported");
}
+ @Override
+ public Partition exchange_partition(Map<String, String> partitionSpecs, String sourceCatName,
+ String sourceDbName, String sourceTableName, String destCatName, String destDbName, String destTableName)
+ throws TException {
+ org.apache.hadoop.hive.metastore.api.Table sourceTempTable = getTempTable(sourceDbName, sourceTableName);
+ org.apache.hadoop.hive.metastore.api.Table destTempTable = getTempTable(destDbName, destTableName);
+ if (sourceTempTable == null && destTempTable == null) {
+ return super
+ .exchange_partition(partitionSpecs, sourceCatName, sourceDbName, sourceTableName, destCatName, destDbName,
+ destTableName);
+ } else if (sourceTempTable != null && destTempTable != null) {
+ TempTable sourceTT = getPartitionedTempTable(sourceTempTable);
+ TempTable destTT = getPartitionedTempTable(destTempTable);
+ List<Partition> partitions = exchangePartitions(partitionSpecs, sourceTempTable, sourceTT, destTempTable, destTT);
+ if (!partitions.isEmpty()) {
+ return partitions.get(0);
+ }
+ }
+ throw new MetaException("Exchanging partitions between temporary and non-temporary tables is not supported.");
+ }
+
+ @Override
+ public List<Partition> exchange_partitions(Map<String, String> partitionSpecs, String sourceCatName,
+ String sourceDbName, String sourceTableName, String destCatName, String destDbName, String destTableName)
+ throws TException {
+ org.apache.hadoop.hive.metastore.api.Table sourceTempTable = getTempTable(sourceDbName, sourceTableName);
+ org.apache.hadoop.hive.metastore.api.Table destTempTable = getTempTable(destDbName, destTableName);
+ if (sourceTempTable == null && destTempTable == null) {
+ return super
+ .exchange_partitions(partitionSpecs, sourceCatName, sourceDbName, sourceTableName, destCatName, destDbName,
+ destTableName);
+ } else if (sourceTempTable != null && destTempTable != null) {
+ return exchangePartitions(partitionSpecs, sourceTempTable, getPartitionedTempTable(sourceTempTable),
+ destTempTable, getPartitionedTempTable(destTempTable));
+ }
+ throw new MetaException("Exchanging partitions between temporary and non-temporary tables is not supported.");
+ }
+
+ private List<Partition> exchangePartitions(Map<String, String> partitionSpecs,
+ org.apache.hadoop.hive.metastore.api.Table sourceTable, TempTable sourceTempTable,
+ org.apache.hadoop.hive.metastore.api.Table destTable, TempTable destTempTable) throws TException {
+ if (partitionSpecs == null || partitionSpecs.isEmpty()) {
+ throw new MetaException("PartitionSpecs cannot be null or empty.");
+ }
+ List<String> partitionVals = getPvals(sourceTable.getPartitionKeys(), partitionSpecs);
+ if (partitionVals.stream().allMatch(String::isEmpty)) {
+ throw new MetaException("Invalid partition key & values; keys " +
+ Arrays.toString(sourceTable.getPartitionKeys().toArray()) + ", values " +
+ Arrays.toString(partitionVals.toArray()));
+ }
+ List<Partition> partitionsToExchange = sourceTempTable
+ .getPartitionsByPartitionVals(partitionVals);
+ if (partitionSpecs == null) {
+ throw new MetaException("The partition specs must be not null.");
+ }
+ if (partitionsToExchange.isEmpty()) {
+ throw new MetaException(
+ "No partition is found with the values " + partitionSpecs + " for the table " + sourceTable.getTableName());
+ }
+
+ boolean sameColumns = compareFieldColumns(sourceTable.getSd().getCols(), destTable.getSd().getCols());
+ boolean samePartitions = compareFieldColumns(sourceTable.getPartitionKeys(), destTable.getPartitionKeys());
+ if (!(sameColumns && samePartitions)) {
+ throw new MetaException("The tables have different schemas. Their partitions cannot be exchanged.");
+ }
+ // Check if any of the partitions already exists in the destTable
+ for (Partition partition : partitionsToExchange) {
+ String partToExchangeName = makePartName(destTable.getPartitionKeys(), partition.getValues());
+ if (destTempTable.getPartition(partToExchangeName) != null) {
+ throw new MetaException(
+ "The partition " + partToExchangeName + " already exists in the table " + destTable.getTableName());
+ }
+ }
+
+ List<Partition> result = new ArrayList<>();
+ for (Partition partition : partitionsToExchange) {
+ Partition destPartition = new Partition(partition);
+ destPartition.setCatName(destTable.getCatName());
+ destPartition.setDbName(destTable.getDbName());
+ destPartition.setTableName(destTable.getTableName());
+ // the destPartition is created from the original partition, therefore all it's properties are copied, including
+ // the location. We must force the rewrite of the location (getPartitionLocation(forceRewrite=true))
+ destPartition.getSd().setLocation(getPartitionLocation(destTable, destPartition, true).toString());
+ wh.renameDir(new Path(partition.getSd().getLocation()), new Path(destPartition.getSd().getLocation()), false);
+ destPartition = destTempTable.addPartition(destPartition);
+ dropPartition(sourceTable.getDbName(), sourceTable.getTableName(), partition.getValues());
+ result.add(destPartition);
+ }
+ return result;
+ }
+
private TempTable getPartitionedTempTable(org.apache.hadoop.hive.metastore.api.Table t) throws MetaException {
String qualifiedTableName = Warehouse.
getQualifiedName(t.getDbName().toLowerCase(), t.getTableName().toLowerCase());
@@ -1677,17 +1773,19 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
*
* @param table the parent table, must be not null
* @param partition instance of the partition, must be not null
+ * @param forceOverwrite force recalculation of location based on table/partition name
* @return location of partition
* @throws MetaException if the partition location cannot be specified or the location is invalid.
*/
- private Path getPartitionLocation(org.apache.hadoop.hive.metastore.api.Table table, Partition partition)
+ private Path getPartitionLocation(org.apache.hadoop.hive.metastore.api.Table table, Partition partition,
+ boolean forceOverwrite)
throws MetaException {
Path partLocation = null;
String partLocationStr = null;
if (partition.getSd() != null) {
partLocationStr = partition.getSd().getLocation();
}
- if (partLocationStr == null || partLocationStr.isEmpty()) {
+ if (partLocationStr == null || partLocationStr.isEmpty() || forceOverwrite) {
// set default location if not specified and this is
// a physical table partition (not a view)
if (table.getSd().getLocation() != null) {
@@ -1732,7 +1830,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
TempTable tt = getPartitionedTempTable(table);
List<Partition> result = tt.addPartitions(deepCopyPartitions(partitions), ifNotExists);
for (Partition p : result) {
- createAndSetLocationForAddedPartition(p, getPartitionLocation(table, p));
+ createAndSetLocationForAddedPartition(p, getPartitionLocation(table, p, false));
}
return result;
}
@@ -1787,7 +1885,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
checkPartitionProperties(p);
// validate partition location
- getPartitionLocation(table, p);
+ getPartitionLocation(table, p, false);
}
return true;
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientExchangePartitionsTempTable.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientExchangePartitionsTempTable.java
new file mode 100644
index 0000000..6844c5d
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientExchangePartitionsTempTable.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.metadata;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.client.TestExchangePartitions;
+import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Test class for exchange partitions related methods on temporary tables.
+ */
+@RunWith(Parameterized.class)
+@Category(MetastoreCheckinTest.class)
+public class TestSessionHiveMetastoreClientExchangePartitionsTempTable extends TestExchangePartitions {
+
+ private HiveConf conf;
+ private Warehouse wh;
+
+ public TestSessionHiveMetastoreClientExchangePartitionsTempTable(String name, AbstractMetaStoreService metaStore) {
+ super(name, metaStore);
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ initHiveConf();
+ wh = new Warehouse(conf);
+ SessionState.start(conf);
+ setClient(Hive.get(conf).getMSC());
+ getClient().dropDatabase(DB_NAME, true, true, true);
+ getMetaStore().cleanWarehouseDirs();
+ createTestTables();
+ }
+
+ private void initHiveConf() throws Exception{
+ conf = Hive.get().getConf();
+ conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true);
+ }
+
+ @Override
+ protected Table createTable(String dbName, String tableName, List<FieldSchema> partCols,
+ List<FieldSchema> cols, String location) throws Exception {
+ new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tableName)
+ .setCols(cols)
+ .setPartCols(partCols)
+ .setLocation(location)
+ .setTemporary(true)
+ .create(getClient(), getMetaStore().getConf());
+ return getClient().getTable(dbName, tableName);
+ }
+
+ @Override
+ @Test(expected = MetaException.class)
+ public void testExchangePartitionsNonExistingPartLocation() throws Exception {
+ Map<String, String> partitionSpecs = getPartitionSpec(getPartitions()[1]);
+ getMetaStore().cleanWarehouseDirs();
+ cleanTempTableDir(getSourceTable());
+ cleanTempTableDir(getDestTable());
+ getClient().exchange_partitions(partitionSpecs, getSourceTable().getDbName(),
+ getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName());
+ }
+
+ @Override
+ @Test
+ public void testExchangePartitionsCustomTableAndPartLocation() throws Exception {
+ Table source = createTable(DB_NAME, "test_source_table_cust_loc",
+ getYearMonthAndDayPartCols(), getMetaStore().getWarehouseRoot() + "/sourceTable");
+ Table dest = createTable(DB_NAME, "test_dest_table_cust_loc", getYearMonthAndDayPartCols(),
+ getMetaStore().getWarehouseRoot() + "/destTable");
+ org.apache.hadoop.hive.metastore.api.Partition[] parts = new Partition[2];
+ parts[0] = createPartition(source, Lists.newArrayList("2019", "may", "11"),
+ source.getSd().getLocation() + "/2019m11");
+ parts[1] = createPartition(source, Lists.newArrayList("2019", "july", "23"),
+ source.getSd().getLocation() + "/2019j23");
+
+ Map<String, String> partitionSpecs = getPartitionSpec(parts[1]);
+ getClient().exchange_partitions(partitionSpecs, source.getDbName(),
+ source.getTableName(), dest.getDbName(), dest.getTableName());
+
+ checkRemainingPartitions(source, dest, Lists.newArrayList(parts[0]));
+ List<Partition> destTablePartitions = getClient().listPartitions(dest.getDbName(), dest.getTableName(), (short) -1);
+ Assert.assertEquals(1, destTablePartitions.size());
+ checkExchangedPartitions(source, dest, Lists.newArrayList(parts[1]));
+ }
+
+ @Override
+ @Test
+ public void testExchangePartitionsCustomPartLocation() throws Exception {
+ Table source = createTable(DB_NAME, "test_source_table", getYearMonthAndDayPartCols(), null);
+ Table dest = createTable(DB_NAME, "test_dest_table", getYearMonthAndDayPartCols(), null);
+ Partition[] parts = new Partition[2];
+ parts[0] = createPartition(source, Lists.newArrayList("2019", "march", "15"),
+ source.getSd().getLocation() + "/2019m15");
+ parts[1] = createPartition(source, Lists.newArrayList("2019", "march", "22"),
+ source.getSd().getLocation() + "/2019m22");
+
+ Map<String, String> partitionSpecs = getPartitionSpec(parts[1]);
+ getClient().exchange_partitions(partitionSpecs, source.getDbName(),
+ source.getTableName(), dest.getDbName(), dest.getTableName());
+
+ checkRemainingPartitions(source, dest, Lists.newArrayList(parts[0]));
+ List<Partition> destTablePartitions = getClient().listPartitions(dest.getDbName(), dest.getTableName(), (short) -1);
+ Assert.assertEquals(1, destTablePartitions.size());
+ checkExchangedPartitions(source, dest, Lists.newArrayList(parts[1]));
+ }
+
+ @Override
+ @Test
+ public void testExchangePartitionsOnlyMonthSetInPartSpec() throws Exception {
+ Map<String, String> partitionSpecs = new HashMap<>();
+ partitionSpecs.put(YEAR_COL_NAME, "");
+ partitionSpecs.put(MONTH_COL_NAME, "march");
+ partitionSpecs.put(DAY_COL_NAME, "");
+
+ getClient().exchange_partitions(partitionSpecs, getSourceTable().getDbName(),
+ getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName());
+ checkRemainingPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[2],
+ getPartitions()[3], getPartitions()[4]));
+ List<Partition> exchangedPartitions = getClient().listPartitions(getDestTable().getDbName(),
+ getDestTable().getTableName(), MAX);
+ Assert.assertEquals(2, exchangedPartitions.size());
+ checkExchangedPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[0],
+ getPartitions()[1]));
+ }
+
+ @Override
+ @Test
+ public void testExchangePartitionsYearAndDaySetInPartSpec() throws Exception {
+ Map<String, String> partitionSpecs = new HashMap<>();
+ partitionSpecs.put(YEAR_COL_NAME, "2017");
+ partitionSpecs.put(MONTH_COL_NAME, "");
+ partitionSpecs.put(DAY_COL_NAME, "22");
+ getClient().exchange_partitions(partitionSpecs, getSourceTable().getDbName(),
+ getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName());
+ checkRemainingPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[0],
+ getPartitions()[2], getPartitions()[3], getPartitions()[4]));
+ List<Partition> exchangedPartitions = getClient().listPartitions(getDestTable().getDbName(),
+ getDestTable().getTableName(), MAX);
+ Assert.assertEquals(1, exchangedPartitions.size());
+ checkExchangedPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[1]));
+ }
+
+ @Override
+ @Test
+ public void testExchangePartition() throws Exception {
+ Map<String, String> partitionSpecs = getPartitionSpec(getPartitions()[1]);
+ Partition exchangedPartition =
+ getClient().exchange_partition(partitionSpecs, getSourceTable().getDbName(),
+ getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName());
+ Assert.assertNotNull(exchangedPartition);
+ checkExchangedPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[1]));
+ checkRemainingPartitions(getSourceTable(), getDestTable(),
+ Lists.newArrayList(getPartitions()[0], getPartitions()[2], getPartitions()[3], getPartitions()[4]));
+ }
+
+ @Override
+ @Test
+ public void testExchangePartitionYearSet() throws Exception {
+ Map<String, String> partitionSpecs = getPartitionSpec(Lists.newArrayList("2017", "", ""));
+ Partition exchangedPartition =
+ getClient().exchange_partition(partitionSpecs, getSourceTable().getDbName(),
+ getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName());
+ Assert.assertNotNull(exchangedPartition);
+ checkExchangedPartitions(getSourceTable(), getDestTable(),
+ Lists.newArrayList(getPartitions()[0], getPartitions()[1], getPartitions()[2], getPartitions()[3]));
+ checkRemainingPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[4]));
+ }
+
+ @Override
+ @Test
+ public void testExchangePartitionCustomTableAndPartLocation() throws Exception {
+ Table source = createTable(DB_NAME, "test_source_table_cust_loc",
+ getYearMonthAndDayPartCols(), getMetaStore().getWarehouseRoot() + "/sourceTable");
+ Table dest = createTable(DB_NAME, "test_dest_table_cust_loc", getYearMonthAndDayPartCols(),
+ getMetaStore().getWarehouseRoot() + "/destTable");
+ Partition[] parts = new Partition[2];
+ parts[0] = createPartition(source, Lists.newArrayList("2019", "may", "11"),
+ source.getSd().getLocation() + "/2019m11");
+ parts[1] = createPartition(source, Lists.newArrayList("2019", "july", "23"),
+ source.getSd().getLocation() + "/2019j23");
+
+ Map<String, String> partitionSpecs = getPartitionSpec(parts[1]);
+ getClient().exchange_partition(partitionSpecs, source.getDbName(),
+ source.getTableName(), dest.getDbName(), dest.getTableName());
+
+ checkRemainingPartitions(source, dest, Lists.newArrayList(parts[0]));
+ List<Partition> destTablePartitions =
+ getClient().listPartitions(dest.getDbName(), dest.getTableName(), (short) -1);
+ Assert.assertEquals(1, destTablePartitions.size());
+ checkExchangedPartitions(source, dest, Lists.newArrayList(parts[1]));
+ }
+
+ @Override
+ @Test
+ public void testExchangePartitionCustomPartLocation() throws Exception {
+ Table source = createTable(DB_NAME, "test_source_table", getYearMonthAndDayPartCols(), null);
+ Table dest = createTable(DB_NAME, "test_dest_table", getYearMonthAndDayPartCols(), null);
+ Partition[] parts = new Partition[2];
+ parts[0] = createPartition(source, Lists.newArrayList("2019", "march", "15"),
+ source.getSd().getLocation() + "/2019m15");
+ parts[1] = createPartition(source, Lists.newArrayList("2019", "march", "22"),
+ source.getSd().getLocation() + "/2019m22");
+
+ Map<String, String> partitionSpecs = getPartitionSpec(parts[1]);
+ getClient().exchange_partition(partitionSpecs, source.getDbName(),
+ source.getTableName(), dest.getDbName(), dest.getTableName());
+ checkRemainingPartitions(source, dest, Lists.newArrayList(parts[0]));
+ List<Partition> destTablePartitions =
+ getClient().listPartitions(dest.getDbName(), dest.getTableName(), (short) -1);
+ Assert.assertEquals(1, destTablePartitions.size());
+ checkExchangedPartitions(source, dest, Lists.newArrayList(parts[1]));
+ }
+
+ @Override
+ @Test(expected = MetaException.class)
+ public void testExchangePartitionNonExistingPartLocation() throws Exception {
+ Map<String, String> partitionSpecs = getPartitionSpec(getPartitions()[1]);
+ cleanTempTableDir(getSourceTable());
+ cleanTempTableDir(getDestTable());
+ getClient().exchange_partition(partitionSpecs, getSourceTable().getDbName(),
+ getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName());
+ }
+
+ @Override
+ @Test
+ public void testExchangePartitionOnlyMonthSetInPartSpec() throws Exception {
+ Map<String, String> partitionSpecs = new HashMap<>();
+ partitionSpecs.put(YEAR_COL_NAME, "");
+ partitionSpecs.put(MONTH_COL_NAME, "march");
+ partitionSpecs.put(DAY_COL_NAME, "");
+ getClient().exchange_partitions(partitionSpecs, getSourceTable().getDbName(),
+ getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName());
+ checkRemainingPartitions(getSourceTable(), getDestTable(),
+ Lists.newArrayList(getPartitions()[2], getPartitions()[3], getPartitions()[4]));
+ List<Partition> exchangedPartitions = getClient().listPartitions(getDestTable().getDbName(),
+ getDestTable().getTableName(), MAX);
+ Assert.assertEquals(2, exchangedPartitions.size());
+ checkExchangedPartitions(getSourceTable(), getDestTable(),
+ Lists.newArrayList(getPartitions()[0], getPartitions()[1]));
+ }
+
+ @Override
+ @Test
+ public void testExchangePartitionYearAndDaySetInPartSpec() throws Exception {
+ Map<String, String> partitionSpecs = new HashMap<>();
+ partitionSpecs.put(YEAR_COL_NAME, "2017");
+ partitionSpecs.put(MONTH_COL_NAME, "");
+ partitionSpecs.put(DAY_COL_NAME, "22");
+ getClient().exchange_partition(partitionSpecs, getSourceTable().getDbName(),
+ getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName());
+ checkRemainingPartitions(getSourceTable(), getDestTable(),
+ Lists.newArrayList(getPartitions()[0], getPartitions()[2], getPartitions()[3], getPartitions()[4]));
+ List<Partition> exchangedPartitions = getClient().listPartitions(getDestTable().getDbName(),
+ getDestTable().getTableName(), MAX);
+ Assert.assertEquals(1, exchangedPartitions.size());
+ checkExchangedPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[1]));
+ }
+
+ @Test(expected = MetaException.class)
+ public void testExchangePartitionBetweenTempAndNonTemp() throws Exception {
+ Table nonTempTable = createNonTempTable(DB_NAME, "nonTempTable", getYearMonthAndDayPartCols(), null);
+ Map<String, String> partitionSpecs = new HashMap<>();
+ partitionSpecs.put(YEAR_COL_NAME, "2017");
+ partitionSpecs.put(MONTH_COL_NAME, "march");
+ partitionSpecs.put(DAY_COL_NAME, "22");
+ getClient().exchange_partition(partitionSpecs, getSourceTable().getDbName(), getSourceTable().getTableName(),
+ nonTempTable.getDbName(), nonTempTable.getTableName());
+ }
+
+ @Test(expected = MetaException.class)
+ public void testExchangePartitionBetweenNonTempAndTemp() throws Exception {
+ Table nonTempTable = createNonTempTable(DB_NAME, "nonTempTable", getYearMonthAndDayPartCols(), null);
+ Map<String, String> partitionSpecs = new HashMap<>();
+ partitionSpecs.put(YEAR_COL_NAME, "2017");
+ partitionSpecs.put(MONTH_COL_NAME, "march");
+ partitionSpecs.put(DAY_COL_NAME, "22");
+ getClient().exchange_partition(partitionSpecs, nonTempTable.getDbName(), nonTempTable.getTableName(),
+ getDestTable().getDbName(), getDestTable().getTableName());
+ }
+
+ @Test(expected = MetaException.class)
+ public void testExchangePartitionsBetweenTempAndNonTemp() throws Exception {
+ Table nonTempTable = createNonTempTable(DB_NAME, "nonTempTable", getYearMonthAndDayPartCols(), null);
+ Map<String, String> partitionSpecs = new HashMap<>();
+ partitionSpecs.put(YEAR_COL_NAME, "2017");
+ partitionSpecs.put(MONTH_COL_NAME, "");
+ partitionSpecs.put(DAY_COL_NAME, "23");
+ getClient().exchange_partitions(partitionSpecs, getSourceTable().getDbName(), getSourceTable().getTableName(),
+ nonTempTable.getDbName(), nonTempTable.getTableName());
+ }
+
+ @Test(expected = MetaException.class)
+ public void testExchangePartitionsBetweenNonTempAndTemp() throws Exception {
+ Table nonTempTable = createNonTempTable(DB_NAME, "nonTempTable", getYearMonthAndDayPartCols(), null);
+ Map<String, String> partitionSpecs = new HashMap<>();
+ partitionSpecs.put(YEAR_COL_NAME, "2017");
+ partitionSpecs.put(MONTH_COL_NAME, "");
+ partitionSpecs.put(DAY_COL_NAME, "23");
+ getClient().exchange_partitions(partitionSpecs, nonTempTable.getDbName(), nonTempTable.getTableName(),
+ getDestTable().getDbName(), getDestTable().getTableName());
+ }
+
+ private Table createNonTempTable(String dbName, String tableName, List<FieldSchema> partCols,
+ String location) throws Exception {
+ List<FieldSchema> cols = new ArrayList<>();
+ cols.add(new FieldSchema("test_id", INT_COL_TYPE, "test col id"));
+ cols.add(new FieldSchema("test_value", "string", "test col value"));
+ new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tableName)
+ .setCols(cols)
+ .setPartCols(partCols)
+ .setLocation(location)
+ .setTemporary(false)
+ .create(getClient(), getMetaStore().getConf());
+ return getClient().getTable(dbName, tableName);
+ }
+
+ private void cleanTempTableDir(Table table) throws MetaException {
+ wh.deleteDir(new Path(table.getSd().getLocation()), true, false, false);
+ }
+
+}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestExchangePartitions.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestExchangePartitions.java
index 1a2b7e4..b008860 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestExchangePartitions.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestExchangePartitions.java
@@ -56,16 +56,16 @@ public class TestExchangePartitions extends MetaStoreClientTest {
private AbstractMetaStoreService metaStore;
private IMetaStoreClient client;
- private static final String DB_NAME = "test_partition_db";
+ protected static final String DB_NAME = "test_partition_db";
private static final String STRING_COL_TYPE = "string";
- private static final String INT_COL_TYPE = "int";
- private static final String YEAR_COL_NAME = "year";
- private static final String MONTH_COL_NAME = "month";
- private static final String DAY_COL_NAME = "day";
- private static final short MAX = -1;
- private static Table sourceTable;
- private static Table destTable;
- private static Partition[] partitions;
+ protected static final String INT_COL_TYPE = "int";
+ protected static final String YEAR_COL_NAME = "year";
+ protected static final String MONTH_COL_NAME = "month";
+ protected static final String DAY_COL_NAME = "day";
+ protected static final short MAX = -1;
+ private Table sourceTable;
+ private Table destTable;
+ private Partition[] partitions;
public TestExchangePartitions(String name, AbstractMetaStoreService metaStore) {
this.metaStore = metaStore;
@@ -79,10 +79,7 @@ public class TestExchangePartitions extends MetaStoreClientTest {
// Clean up the database
client.dropDatabase(DB_NAME, true, true, true);
metaStore.cleanWarehouseDirs();
- createDB(DB_NAME);
- sourceTable = createSourceTable();
- destTable = createDestTable();
- partitions = createTestPartitions();
+ createTestTables();
}
@After
@@ -100,6 +97,30 @@ public class TestExchangePartitions extends MetaStoreClientTest {
}
}
+ protected IMetaStoreClient getClient() {
+ return client;
+ }
+
+ protected void setClient(IMetaStoreClient client) {
+ this.client = client;
+ }
+
+ protected AbstractMetaStoreService getMetaStore() {
+ return metaStore;
+ }
+
+ protected Table getSourceTable() {
+ return sourceTable;
+ }
+
+ protected Table getDestTable() {
+ return destTable;
+ }
+
+ protected Partition[] getPartitions() {
+ return partitions;
+ }
+
// Tests for the List<Partition> exchange_partitions(Map<String, String> partitionSpecs, String
// sourceDb, String sourceTable, String destdb, String destTableName) method
@@ -1164,6 +1185,13 @@ public class TestExchangePartitions extends MetaStoreClientTest {
}
// Helper methods
+ protected void createTestTables() throws Exception {
+ createDB(DB_NAME);
+ sourceTable = createSourceTable();
+ destTable = createDestTable();
+ partitions = createTestPartitions();
+ }
+
private void createDB(String dbName) throws TException {
new DatabaseBuilder()
.setName(dbName)
@@ -1178,7 +1206,7 @@ public class TestExchangePartitions extends MetaStoreClientTest {
return createTable(DB_NAME, "test_part_exch_table_dest", getYearMonthAndDayPartCols(), null);
}
- private Table createTable(String dbName, String tableName, List<FieldSchema> partCols,
+ protected Table createTable(String dbName, String tableName, List<FieldSchema> partCols,
String location) throws Exception {
List<FieldSchema> cols = new ArrayList<>();
cols.add(new FieldSchema("test_id", INT_COL_TYPE, "test col id"));
@@ -1186,7 +1214,7 @@ public class TestExchangePartitions extends MetaStoreClientTest {
return createTable(dbName, tableName, partCols, cols, location);
}
- private Table createTable(String dbName, String tableName, List<FieldSchema> partCols,
+ protected Table createTable(String dbName, String tableName, List<FieldSchema> partCols,
List<FieldSchema> cols, String location) throws Exception {
new TableBuilder()
.setDbName(dbName)
@@ -1226,7 +1254,7 @@ public class TestExchangePartitions extends MetaStoreClientTest {
return parts;
}
- private Partition createPartition(Table table, List<String> values, String location)
+ protected Partition createPartition(Table table, List<String> values, String location)
throws Exception {
Partition partition = buildPartition(table, values, location);
client.add_partition(partition);
@@ -1249,7 +1277,7 @@ public class TestExchangePartitions extends MetaStoreClientTest {
return partition;
}
- private void checkExchangedPartitions(Table sourceTable, Table destTable,
+ protected void checkExchangedPartitions(Table sourceTable, Table destTable,
List<Partition> partitions) throws Exception {
for (Partition partition : partitions) {
@@ -1286,7 +1314,7 @@ public class TestExchangePartitions extends MetaStoreClientTest {
}
}
- private void checkRemainingPartitions(Table sourceTable, Table destTable,
+ protected void checkRemainingPartitions(Table sourceTable, Table destTable,
List<Partition> partitions) throws Exception {
for (Partition partition : partitions) {
@@ -1312,11 +1340,11 @@ public class TestExchangePartitions extends MetaStoreClientTest {
}
}
- private static Map<String, String> getPartitionSpec(Partition partition) {
+ protected static Map<String, String> getPartitionSpec(Partition partition) {
return getPartitionSpec(partition.getValues());
}
- private static Map<String, String> getPartitionSpec(List<String> values) {
+ protected static Map<String, String> getPartitionSpec(List<String> values) {
Map<String, String> partitionSpecs = new HashMap<>();
List<FieldSchema> partCols = getYearMonthAndDayPartCols();
for (int i = 0; i < partCols.size(); i++) {
@@ -1327,7 +1355,7 @@ public class TestExchangePartitions extends MetaStoreClientTest {
return partitionSpecs;
}
- private static List<FieldSchema> getYearMonthAndDayPartCols() {
+ protected static List<FieldSchema> getYearMonthAndDayPartCols() {
List<FieldSchema> cols = new ArrayList<>();
cols.add(new FieldSchema(YEAR_COL_NAME, STRING_COL_TYPE, "year part col"));
cols.add(new FieldSchema(MONTH_COL_NAME, STRING_COL_TYPE, "month part col"));