You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/10/04 16:27:07 UTC
svn commit: r1394087 - in /hive/trunk:
metastore/src/java/org/apache/hadoop/hive/metastore/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: namit
Date: Thu Oct 4 14:27:06 2012
New Revision: 1394087
URL: http://svn.apache.org/viewvc?rev=1394087&view=rev
Log:
HIVE-2874 Renaming external partition changes location
(Zhenxiao Luo via namit)
Added:
hive/trunk/ql/src/test/queries/clientpositive/rename_external_partition_location.q
hive/trunk/ql/src/test/results/clientpositive/rename_external_partition_location.q.out
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java?rev=1394087&r1=1394086&r2=1394087&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java Thu Oct 4 14:27:06 2012
@@ -287,48 +287,55 @@ public class HiveAlterHandler implements
throw new InvalidObjectException(
"Unable to rename partition because table or database do not exist");
}
- try {
- destPath = new Path(wh.getTablePath(msdb.getDatabase(dbname), name),
+
+ // if the external partition is renamed, the file should not change
+ if (tbl.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
+ new_part.getSd().setLocation(oldPart.getSd().getLocation());
+ msdb.alterPartition(dbname, name, part_vals, new_part);
+ } else {
+ try {
+ destPath = new Path(wh.getTablePath(msdb.getDatabase(dbname), name),
Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()));
- destPath = constructRenamedPartitionPath(destPath, new_part);
- } catch (NoSuchObjectException e) {
- LOG.debug(e);
- throw new InvalidOperationException(
+ destPath = constructRenamedPartitionPath(destPath, new_part);
+ } catch (NoSuchObjectException e) {
+ LOG.debug(e);
+ throw new InvalidOperationException(
"Unable to change partition or table. Database " + dbname + " does not exist"
- + " Check metastore logs for detailed stack." + e.getMessage());
- }
- if (destPath != null) {
- newPartLoc = destPath.toString();
- oldPartLoc = oldPart.getSd().getLocation();
-
- srcPath = new Path(oldPartLoc);
-
- LOG.info("srcPath:" + oldPartLoc);
- LOG.info("descPath:" + newPartLoc);
- srcFs = wh.getFs(srcPath);
- destFs = wh.getFs(destPath);
- // check that src and dest are on the same file system
- if (srcFs != destFs) {
- throw new InvalidOperationException("table new location " + destPath
+ + " Check metastore logs for detailed stack." + e.getMessage());
+ }
+ if (destPath != null) {
+ newPartLoc = destPath.toString();
+ oldPartLoc = oldPart.getSd().getLocation();
+
+ srcPath = new Path(oldPartLoc);
+
+ LOG.info("srcPath:" + oldPartLoc);
+ LOG.info("descPath:" + newPartLoc);
+ srcFs = wh.getFs(srcPath);
+ destFs = wh.getFs(destPath);
+ // check that src and dest are on the same file system
+ if (srcFs != destFs) {
+ throw new InvalidOperationException("table new location " + destPath
+ " is on a different file system than the old location "
+ srcPath + ". This operation is not supported");
- }
- try {
- srcFs.exists(srcPath); // check that src exists and also checks
- if (newPartLoc.compareTo(oldPartLoc) != 0 && destFs.exists(destPath)) {
- throw new InvalidOperationException("New location for this table "
+ }
+ try {
+ srcFs.exists(srcPath); // check that src exists and also checks
+ if (newPartLoc.compareTo(oldPartLoc) != 0 && destFs.exists(destPath)) {
+ throw new InvalidOperationException("New location for this table "
+ tbl.getDbName() + "." + tbl.getTableName()
+ " already exists : " + destPath);
- }
- } catch (IOException e) {
- Warehouse.closeFs(srcFs);
- Warehouse.closeFs(destFs);
- throw new InvalidOperationException("Unable to access new location "
+ }
+ } catch (IOException e) {
+ Warehouse.closeFs(srcFs);
+ Warehouse.closeFs(destFs);
+ throw new InvalidOperationException("Unable to access new location "
+ destPath + " for partition " + tbl.getDbName() + "."
+ tbl.getTableName() + " " + new_part.getValues());
+ }
+ new_part.getSd().setLocation(newPartLoc);
+ msdb.alterPartition(dbname, name, part_vals, new_part);
}
- new_part.getSd().setLocation(newPartLoc);
- msdb.alterPartition(dbname, name, part_vals, new_part);
}
success = msdb.commitTransaction();
@@ -336,7 +343,7 @@ public class HiveAlterHandler implements
if (!success) {
msdb.rollbackTransaction();
}
- if (success && newPartLoc.compareTo(oldPartLoc) != 0) {
+ if (success && newPartLoc != null && newPartLoc.compareTo(oldPartLoc) != 0) {
//rename the data directory
try{
if (srcFs.exists(srcPath)) {
Added: hive/trunk/ql/src/test/queries/clientpositive/rename_external_partition_location.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/rename_external_partition_location.q?rev=1394087&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/rename_external_partition_location.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/rename_external_partition_location.q Thu Oct 4 14:27:06 2012
@@ -0,0 +1,24 @@
+
+dfs -mkdir ${system:test.tmp.dir}/ex_table;
+
+CREATE EXTERNAL TABLE ex_table ( key INT, value STRING)
+ PARTITIONED BY (part STRING)
+ STORED AS textfile
+ LOCATION 'file:${system:test.tmp.dir}/ex_table';
+
+INSERT OVERWRITE TABLE ex_table PARTITION (part='part1')
+SELECT key, value FROM src WHERE key < 10;
+
+SHOW PARTITIONS ex_table;
+SELECT * from ex_table where part='part1' ORDER BY key;
+
+dfs -ls ${system:test.tmp.dir}/ex_table/part=part1;
+dfs -cat ${system:test.tmp.dir}/ex_table/part=part1/000000_0;
+
+ALTER TABLE ex_table PARTITION (part='part1') RENAME TO PARTITION (part='part2');
+
+SHOW PARTITIONS ex_table;
+SELECT * from ex_table where part='part2' ORDER BY key;
+
+dfs -ls ${system:test.tmp.dir}/ex_table/part=part1;
+dfs -cat ${system:test.tmp.dir}/ex_table/part=part1/000000_0;
Added: hive/trunk/ql/src/test/results/clientpositive/rename_external_partition_location.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/rename_external_partition_location.q.out?rev=1394087&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/rename_external_partition_location.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/rename_external_partition_location.q.out Thu Oct 4 14:27:06 2012
@@ -0,0 +1,113 @@
+PREHOOK: query: CREATE EXTERNAL TABLE ex_table ( key INT, value STRING)
+ PARTITIONED BY (part STRING)
+ STORED AS textfile
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE ex_table ( key INT, value STRING)
+ PARTITIONED BY (part STRING)
+ STORED AS textfile
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@ex_table
+PREHOOK: query: INSERT OVERWRITE TABLE ex_table PARTITION (part='part1')
+SELECT key, value FROM src WHERE key < 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ex_table@part=part1
+POSTHOOK: query: INSERT OVERWRITE TABLE ex_table PARTITION (part='part1')
+SELECT key, value FROM src WHERE key < 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ex_table@part=part1
+POSTHOOK: Lineage: ex_table PARTITION(part=part1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ex_table PARTITION(part=part1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SHOW PARTITIONS ex_table
+PREHOOK: type: SHOWPARTITIONS
+POSTHOOK: query: SHOW PARTITIONS ex_table
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Lineage: ex_table PARTITION(part=part1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ex_table PARTITION(part=part1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+part=part1
+PREHOOK: query: SELECT * from ex_table where part='part1' ORDER BY key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ex_table@part=part1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from ex_table where part='part1' ORDER BY key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ex_table@part=part1
+#### A masked pattern was here ####
+POSTHOOK: Lineage: ex_table PARTITION(part=part1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ex_table PARTITION(part=part1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+0 val_0 part1
+0 val_0 part1
+0 val_0 part1
+2 val_2 part1
+4 val_4 part1
+5 val_5 part1
+5 val_5 part1
+5 val_5 part1
+8 val_8 part1
+9 val_9 part1
+Found 1 items
+#### A masked pattern was here ####
+0val_0
+4val_4
+8val_8
+0val_0
+0val_0
+5val_5
+5val_5
+2val_2
+5val_5
+9val_9
+PREHOOK: query: ALTER TABLE ex_table PARTITION (part='part1') RENAME TO PARTITION (part='part2')
+PREHOOK: type: ALTERTABLE_RENAMEPART
+PREHOOK: Input: default@ex_table
+PREHOOK: Output: default@ex_table@part=part1
+POSTHOOK: query: ALTER TABLE ex_table PARTITION (part='part1') RENAME TO PARTITION (part='part2')
+POSTHOOK: type: ALTERTABLE_RENAMEPART
+POSTHOOK: Input: default@ex_table
+POSTHOOK: Input: default@ex_table@part=part1
+POSTHOOK: Output: default@ex_table@part=part1
+POSTHOOK: Output: default@ex_table@part=part2
+POSTHOOK: Lineage: ex_table PARTITION(part=part1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ex_table PARTITION(part=part1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SHOW PARTITIONS ex_table
+PREHOOK: type: SHOWPARTITIONS
+POSTHOOK: query: SHOW PARTITIONS ex_table
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Lineage: ex_table PARTITION(part=part1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ex_table PARTITION(part=part1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+part=part2
+PREHOOK: query: SELECT * from ex_table where part='part2' ORDER BY key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ex_table@part=part2
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from ex_table where part='part2' ORDER BY key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ex_table@part=part2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: ex_table PARTITION(part=part1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ex_table PARTITION(part=part1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+0 val_0 part2
+0 val_0 part2
+0 val_0 part2
+2 val_2 part2
+4 val_4 part2
+5 val_5 part2
+5 val_5 part2
+5 val_5 part2
+8 val_8 part2
+9 val_9 part2
+Found 1 items
+#### A masked pattern was here ####
+0val_0
+4val_4
+8val_8
+0val_0
+0val_0
+5val_5
+5val_5
+2val_2
+5val_5
+9val_9