You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2017/04/21 18:13:57 UTC
hive git commit: HIVE-16287: Alter table partition rename with
location - moves partition back to hive warehouse (Vihang Karajgaonkarm,
reviewed by Sergio Pena, Ying Chen)
Repository: hive
Updated Branches:
refs/heads/branch-1 bf5a718b3 -> d76b54443
HIVE-16287: Alter table partition rename with location - moves partition back to hive warehouse (Vihang Karajgaonkarm, reviewed by Sergio Pena, Ying Chen)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d76b5444
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d76b5444
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d76b5444
Branch: refs/heads/branch-1
Commit: d76b54443c784114043e4fef89f0fb1738d8b3bb
Parents: bf5a718
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Fri Apr 21 13:13:15 2017 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Fri Apr 21 13:13:15 2017 -0500
----------------------------------------------------------------------
.../apache/hive/hcatalog/cli/TestPermsGrp.java | 6 +-
.../hadoop/hive/metastore/HiveAlterHandler.java | 5 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 2 +-
.../apache/hadoop/hive/metastore/Warehouse.java | 71 +++++++++++++++++++-
.../hive/ql/parse/ImportSemanticAnalyzer.java | 6 +-
.../hadoop/hive/ql/parse/TaskCompiler.java | 2 +-
.../hadoop/hive/ql/metadata/TestHive.java | 2 +-
.../clientpositive/rename_partition_location.q | 14 ++++
.../rename_partition_location.q.out | 24 +++++++
9 files changed, 119 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d76b5444/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java
index 41c6b16..52ad97c 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java
@@ -116,7 +116,7 @@ public class TestPermsGrp extends TestCase {
Table tbl = getTable(dbName, tblName, typeName);
msc.createTable(tbl);
Database db = Hive.get(hcatConf).getDatabase(dbName);
- Path dfsPath = clientWH.getTablePath(db, tblName);
+ Path dfsPath = clientWH.getDefaultTablePath(db, tblName);
cleanupTbl(dbName, tblName, typeName);
// Next user did specify perms.
@@ -126,7 +126,7 @@ public class TestPermsGrp extends TestCase {
assertTrue(e instanceof ExitException);
assertEquals(((ExitException) e).getStatus(), 0);
}
- dfsPath = clientWH.getTablePath(db, tblName);
+ dfsPath = clientWH.getDefaultTablePath(db, tblName);
assertTrue(dfsPath.getFileSystem(hcatConf).getFileStatus(dfsPath).getPermission().equals(FsPermission.valueOf("drwx-wx---")));
cleanupTbl(dbName, tblName, typeName);
@@ -141,7 +141,7 @@ public class TestPermsGrp extends TestCase {
assertTrue(me instanceof ExitException);
}
// No physical dir gets created.
- dfsPath = clientWH.getTablePath(db, tblName);
+ dfsPath = clientWH.getDefaultTablePath(db, tblName);
try {
dfsPath.getFileSystem(hcatConf).getFileStatus(dfsPath);
assert false;
http://git-wip-us.apache.org/repos/asf/hive/blob/d76b5444/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index c059f5e..a32f03d 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -397,8 +397,9 @@ public class HiveAlterHandler implements AlterHandler {
msdb.alterPartition(dbname, name, part_vals, new_part);
} else {
try {
- destPath = new Path(wh.getTablePath(msdb.getDatabase(dbname), name),
- Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()));
+ // if tbl location is available use it
+ // else derive the tbl location from database location
+ destPath = wh.getPartitionPath(msdb.getDatabase(dbname), tbl, new_part.getValues());
destPath = constructRenamedPath(destPath, new Path(new_part.getSd().getLocation()));
} catch (NoSuchObjectException e) {
LOG.debug(e);
http://git-wip-us.apache.org/repos/asf/hive/blob/d76b5444/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 7a2333d..7baa0e6 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -1386,7 +1386,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if (!TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) {
if (tbl.getSd().getLocation() == null
|| tbl.getSd().getLocation().isEmpty()) {
- tblPath = wh.getTablePath(
+ tblPath = wh.getDefaultTablePath(
ms.getDatabase(tbl.getDbName()), tbl.getTableName());
} else {
if (!isExternal(tbl) && !MetaStoreUtils.isNonNativeTable(tbl)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/d76b5444/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
index 011d4d8..a8ef85e 100755
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
@@ -179,7 +179,15 @@ public class Warehouse {
return new Path(getWhRoot(), dbName.toLowerCase() + DATABASE_WAREHOUSE_SUFFIX);
}
- public Path getTablePath(Database db, String tableName)
+ /**
+ * Returns the default location of the table path using the parent database's location
+ *
+ * @param db Database where the table is created
+ * @param tableName table name
+ * @return
+ * @throws MetaException
+ */
+ public Path getDefaultTablePath(Database db, String tableName)
throws MetaException {
return getDnsPath(new Path(getDatabasePath(db), tableName.toLowerCase()));
}
@@ -461,7 +469,7 @@ public class Warehouse {
public Path getPartitionPath(Database db, String tableName,
LinkedHashMap<String, String> pm) throws MetaException {
- return new Path(getTablePath(db, tableName), makePartPath(pm));
+ return new Path(getDefaultTablePath(db, tableName), makePartPath(pm));
}
public Path getPartitionPath(Path tblPath, LinkedHashMap<String, String> pm)
@@ -469,6 +477,65 @@ public class Warehouse {
return new Path(tblPath, makePartPath(pm));
}
+ /**
+ * Returns the default partition path of a table within a given database and partition key value
+ * pairs. It uses the database location and appends it the table name and the partition key,value
+ * pairs to create the Path for the partition directory
+ *
+ * @param db - parent database which is used to get the base location of the partition directory
+ * @param tableName - table name for the partitions
+ * @param pm - Partition key value pairs
+ * @return
+ * @throws MetaException
+ */
+ public Path getDefaultPartitionPath(Database db, String tableName, Map<String, String> pm)
+ throws MetaException {
+ return getPartitionPath(getDefaultTablePath(db, tableName), pm);
+ }
+
+ /**
+ * Returns the path object for the given partition key-value pairs and the base location
+ *
+ * @param tblPath - the base location for the partitions. Typically the table location
+ * @param pm - Partition key value pairs
+ * @return
+ * @throws MetaException
+ */
+ public Path getPartitionPath(Path tblPath, Map<String, String> pm) throws MetaException {
+ return new Path(tblPath, makePartPath(pm));
+ }
+
+ /**
+ * Given a database, a table and the partition key value pairs this method returns the Path object
+ * corresponding to the partition key value pairs. It uses the table location if available else
+ * uses the database location for constructing the path corresponding to the partition key-value
+ * pairs
+ *
+ * @param db - Parent database of the given table
+ * @param table - Table for which the partition key-values are given
+ * @param vals - List of values for the partition keys
+ * @return Path corresponding to the partition key-value pairs
+ * @throws MetaException
+ */
+ public Path getPartitionPath(Database db, Table table, List<String> vals) throws MetaException {
+ List<FieldSchema> partKeys = table.getPartitionKeys();
+ if (partKeys == null || (partKeys.size() != vals.size())) {
+ throw new MetaException("Invalid number of partition keys found for " + table.getTableName());
+ }
+ Map<String, String> pm = new LinkedHashMap<>(vals.size());
+ int i = 0;
+ for (FieldSchema key : partKeys) {
+ pm.put(key.getName(), vals.get(i));
+ i++;
+ }
+
+ if (table.getSd().getLocation() != null) {
+ return getPartitionPath(getDnsPath(new Path(table.getSd().getLocation())), pm);
+ } else {
+ return getDefaultPartitionPath(db, table.getTableName(), pm);
+ }
+ }
+
public boolean isDir(Path f) throws MetaException {
FileSystem fs = null;
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/d76b5444/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 88c4b95..5a761b2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -423,7 +423,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
} else {
Database parentDb = db.getDatabase(tblDesc.getDatabaseName());
tgtPath = new Path(
- wh.getTablePath( parentDb, tblDesc.getTableName()),
+ wh.getDefaultTablePath( parentDb, tblDesc.getTableName()),
Warehouse.makePartPath(partSpec.getPartSpec()));
}
} else {
@@ -738,7 +738,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
if (tblDesc.getLocation() != null) {
tablePath = new Path(tblDesc.getLocation());
} else {
- tablePath = wh.getTablePath(parentDb, tblDesc.getTableName());
+ tablePath = wh.getDefaultTablePath(parentDb, tblDesc.getTableName());
}
checkTargetLocationEmpty(fs, tablePath, replicationSpec);
t.addDependentTask(loadTable(fromURI, table, false));
@@ -781,7 +781,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException(ErrorMsg.DATABASE_NOT_EXISTS.getMsg(tblDesc.getDatabaseName()));
}
if (tblDesc.getLocation() == null) {
- tblDesc.setLocation(wh.getTablePath(parentDb, tblDesc.getTableName()).toString());
+ tblDesc.setLocation(wh.getDefaultTablePath(parentDb, tblDesc.getTableName()).toString());
}
/* Note: In the following section, Metadata-only import handling logic is
http://git-wip-us.apache.org/repos/asf/hive/blob/d76b5444/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index ce7aeb8..ff4f93a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -183,7 +183,7 @@ public abstract class TaskCompiler {
+ " does not exist.");
}
Warehouse wh = new Warehouse(conf);
- targetPath = wh.getTablePath(db.getDatabase(names[0]), names[1]);
+ targetPath = wh.getDefaultTablePath(db.getDatabase(names[0]), names[1]);
} catch (HiveException e) {
throw new SemanticException(e);
} catch (MetaException e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/d76b5444/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
index 99fbd5d..b8159cb 100755
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
@@ -307,7 +307,7 @@ public class TestHive extends TestCase {
assertEquals("Table retention didn't match for table: " + tableName,
tbl.getRetention(), ft.getRetention());
assertEquals("Data location is not set correctly",
- wh.getTablePath(hm.getDatabase(DEFAULT_DATABASE_NAME), tableName).toString(),
+ wh.getDefaultTablePath(hm.getDatabase(DEFAULT_DATABASE_NAME), tableName).toString(),
ft.getDataLocation().toString());
// now that URI and times are set correctly, set the original table's uri and times
// and then compare the two tables
http://git-wip-us.apache.org/repos/asf/hive/blob/d76b5444/ql/src/test/queries/clientpositive/rename_partition_location.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/rename_partition_location.q b/ql/src/test/queries/clientpositive/rename_partition_location.q
index ee4ff81..583c10a 100644
--- a/ql/src/test/queries/clientpositive/rename_partition_location.q
+++ b/ql/src/test/queries/clientpositive/rename_partition_location.q
@@ -17,4 +17,18 @@ SELECT count(*) FROM rename_partition_table where part = '2';
SET hive.exec.post.hooks=;
+CREATE TABLE rename_partition_table_2 (key STRING, value STRING) PARTITIONED BY (part STRING)
+LOCATION '${system:test.tmp.dir}/rename_partition_table_2';
+
+INSERT OVERWRITE TABLE rename_partition_table_2 PARTITION (part = '1') SELECT * FROM src;
+
+ALTER TABLE rename_partition_table_2 PARTITION (part = '1') RENAME TO PARTITION (part = '2');
+
+SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyPartitionIsSubdirectoryOfTableHook;
+
+SELECT count(*) FROM rename_partition_table_2 where part = '2';
+
+SET hive.exec.post.hooks=;
+
DROP TABLE rename_partition_table;
+DROP TABLE rename_partition_table_2;
http://git-wip-us.apache.org/repos/asf/hive/blob/d76b5444/ql/src/test/results/clientpositive/rename_partition_location.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/rename_partition_location.q.out b/ql/src/test/results/clientpositive/rename_partition_location.q.out
index b83bdd0..0552fab 100644
--- a/ql/src/test/results/clientpositive/rename_partition_location.q.out
+++ b/ql/src/test/results/clientpositive/rename_partition_location.q.out
@@ -53,7 +53,31 @@ PREHOOK: Input: default@rename_partition_table
PREHOOK: Input: default@rename_partition_table@part=2
#### A masked pattern was here ####
500
+PREHOOK: query: CREATE TABLE rename_partition_table_2 (key STRING, value STRING) PARTITIONED BY (part STRING)
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@rename_partition_table_2
+PREHOOK: query: INSERT OVERWRITE TABLE rename_partition_table_2 PARTITION (part = '1') SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@rename_partition_table_2@part=1
+PREHOOK: query: ALTER TABLE rename_partition_table_2 PARTITION (part = '1') RENAME TO PARTITION (part = '2')
+PREHOOK: type: ALTERTABLE_RENAMEPART
+PREHOOK: Input: default@rename_partition_table_2
+PREHOOK: Output: default@rename_partition_table_2@part=1
+PREHOOK: query: SELECT count(*) FROM rename_partition_table_2 where part = '2'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@rename_partition_table_2
+PREHOOK: Input: default@rename_partition_table_2@part=2
+#### A masked pattern was here ####
+500
PREHOOK: query: DROP TABLE rename_partition_table
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@rename_partition_table
PREHOOK: Output: default@rename_partition_table
+PREHOOK: query: DROP TABLE rename_partition_table_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@rename_partition_table_2
+PREHOOK: Output: default@rename_partition_table_2