You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2014/08/07 22:57:11 UTC
svn commit: r1616593 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/common/
metastore/src/java/org/apache/hadoop/hive/metastore/
ql/src/java/org/apache/hadoop/hive/ql/metadata/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpos...
Author: szehon
Date: Thu Aug 7 20:57:10 2014
New Revision: 1616593
URL: http://svn.apache.org/r1616593
Log:
HIVE-7441 : Custom partition scheme gets rewritten with hive scheme upon concatenate (Chaoyu Tang via Szehon)
Added:
hive/trunk/ql/src/test/queries/clientpositive/alter_merge_3.q
hive/trunk/ql/src/test/results/clientpositive/alter_merge_3.q.out
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java?rev=1616593&r1=1616592&r2=1616593&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java Thu Aug 7 20:57:10 2014
@@ -613,4 +613,19 @@ public final class FileUtils {
return false;
}
}
+
+ /**
+ * @param fs1
+ * @param fs2
+ * @return return true if both file system arguments point to same file system
+ */
+ public static boolean equalsFileSystem(FileSystem fs1, FileSystem fs2) {
+ //When file system cache is disabled, you get different FileSystem objects
+ // for same file system, so '==' can't be used in such cases
+ //FileSystem api doesn't have a .equals() function implemented, so using
+ //the uri for comparison. FileSystem already uses uri+Configuration for
+ //equality in its CACHE .
+ //Once equality has been added in HDFS-4321, we should make use of it
+ return fs1.getUri().equals(fs2.getUri());
+ }
}
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java?rev=1616593&r1=1616592&r2=1616593&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java Thu Aug 7 20:57:10 2014
@@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -156,7 +157,7 @@ public class HiveAlterHandler implements
destPath = new Path(newTblLoc);
destFs = wh.getFs(destPath);
// check that src and dest are on the same file system
- if (! equalsFileSystem(srcFs, destFs)) {
+ if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
throw new InvalidOperationException("table new location " + destPath
+ " is on a different file system than the old location "
+ srcPath + ". This operation is not supported");
@@ -251,21 +252,6 @@ public class HiveAlterHandler implements
}
}
- /**
- * @param fs1
- * @param fs2
- * @return return true if both file system arguments point to same file system
- */
- private boolean equalsFileSystem(FileSystem fs1, FileSystem fs2) {
- //When file system cache is disabled, you get different FileSystem objects
- // for same file system, so '==' can't be used in such cases
- //FileSystem api doesn't have a .equals() function implemented, so using
- //the uri for comparison. FileSystem already uses uri+Configuration for
- //equality in its CACHE .
- //Once equality has been added in HDFS-4321, we should make use of it
- return fs1.getUri().equals(fs2.getUri());
- }
-
public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
final String name, final List<String> part_vals, final Partition new_part)
throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1616593&r1=1616592&r2=1616593&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Thu Aug 7 20:57:10 2014
@@ -1242,7 +1242,7 @@ public class Hive {
*/
FileSystem oldPartPathFS = oldPartPath.getFileSystem(getConf());
FileSystem loadPathFS = loadPath.getFileSystem(getConf());
- if (oldPartPathFS.equals(loadPathFS)) {
+ if (FileUtils.equalsFileSystem(oldPartPathFS,loadPathFS)) {
newPartPath = oldPartPath;
}
}
Added: hive/trunk/ql/src/test/queries/clientpositive/alter_merge_3.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/alter_merge_3.q?rev=1616593&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/alter_merge_3.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/alter_merge_3.q Thu Aug 7 20:57:10 2014
@@ -0,0 +1,23 @@
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/alter_merge_location/ds20140804;
+dfs -put ../../data/files/smbbucket_1.rc ${system:test.tmp.dir}/alter_merge_location/ds20140804;
+dfs -put ../../data/files/smbbucket_2.rc ${system:test.tmp.dir}/alter_merge_location/ds20140804;
+dfs -put ../../data/files/smbbucket_3.rc ${system:test.tmp.dir}/alter_merge_location/ds20140804;
+
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/alter_merge_location/ds20140805;
+dfs -put ../../data/files/smbbucket_1.rc ${system:test.tmp.dir}/alter_merge_location/ds20140805;
+dfs -put ../../data/files/smbbucket_2.rc ${system:test.tmp.dir}/alter_merge_location/ds20140805;
+dfs -put ../../data/files/smbbucket_3.rc ${system:test.tmp.dir}/alter_merge_location/ds20140805;
+
+create table src_rc_merge_test_part (key int, value string) partitioned by (ds string) stored as rcfile;
+
+set fs.hdfs.impl.disable.cache=false;
+alter table src_rc_merge_test_part add partition (ds = '2014-08-04') location '${system:test.tmp.dir}/alter_merge_location/ds20140804';
+alter table src_rc_merge_test_part partition (ds = '2014-08-04') concatenate;
+select count(1) from src_rc_merge_test_part where ds='2014-08-04';
+
+set fs.hdfs.impl.disable.cache=true;
+alter table src_rc_merge_test_part add partition (ds = '2014-08-05') location '${system:test.tmp.dir}/alter_merge_location/ds20140805';
+alter table src_rc_merge_test_part partition (ds = '2014-08-05') concatenate;
+select count(1) from src_rc_merge_test_part where ds='2014-08-05';
+
+drop table src_rc_merge_test_part;
\ No newline at end of file
Added: hive/trunk/ql/src/test/results/clientpositive/alter_merge_3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/alter_merge_3.q.out?rev=1616593&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/alter_merge_3.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/alter_merge_3.q.out Thu Aug 7 20:57:10 2014
@@ -0,0 +1,71 @@
+PREHOOK: query: create table src_rc_merge_test_part (key int, value string) partitioned by (ds string) stored as rcfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table src_rc_merge_test_part (key int, value string) partitioned by (ds string) stored as rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_rc_merge_test_part
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+PREHOOK: Output: default@src_rc_merge_test_part
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+POSTHOOK: Output: default@src_rc_merge_test_part
+POSTHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-04
+PREHOOK: query: alter table src_rc_merge_test_part partition (ds = '2014-08-04') concatenate
+PREHOOK: type: ALTER_PARTITION_MERGE
+PREHOOK: Input: default@src_rc_merge_test_part
+PREHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-04
+POSTHOOK: query: alter table src_rc_merge_test_part partition (ds = '2014-08-04') concatenate
+POSTHOOK: type: ALTER_PARTITION_MERGE
+POSTHOOK: Input: default@src_rc_merge_test_part
+POSTHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-04
+PREHOOK: query: select count(1) from src_rc_merge_test_part where ds='2014-08-04'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_rc_merge_test_part
+PREHOOK: Input: default@src_rc_merge_test_part@ds=2014-08-04
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from src_rc_merge_test_part where ds='2014-08-04'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_rc_merge_test_part
+POSTHOOK: Input: default@src_rc_merge_test_part@ds=2014-08-04
+#### A masked pattern was here ####
+15
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+PREHOOK: Output: default@src_rc_merge_test_part
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+POSTHOOK: Output: default@src_rc_merge_test_part
+POSTHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-05
+PREHOOK: query: alter table src_rc_merge_test_part partition (ds = '2014-08-05') concatenate
+PREHOOK: type: ALTER_PARTITION_MERGE
+PREHOOK: Input: default@src_rc_merge_test_part
+PREHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-05
+POSTHOOK: query: alter table src_rc_merge_test_part partition (ds = '2014-08-05') concatenate
+POSTHOOK: type: ALTER_PARTITION_MERGE
+POSTHOOK: Input: default@src_rc_merge_test_part
+POSTHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-05
+PREHOOK: query: select count(1) from src_rc_merge_test_part where ds='2014-08-05'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_rc_merge_test_part
+PREHOOK: Input: default@src_rc_merge_test_part@ds=2014-08-05
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from src_rc_merge_test_part where ds='2014-08-05'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_rc_merge_test_part
+POSTHOOK: Input: default@src_rc_merge_test_part@ds=2014-08-05
+#### A masked pattern was here ####
+15
+PREHOOK: query: drop table src_rc_merge_test_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@src_rc_merge_test_part
+PREHOOK: Output: default@src_rc_merge_test_part
+POSTHOOK: query: drop table src_rc_merge_test_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@src_rc_merge_test_part
+POSTHOOK: Output: default@src_rc_merge_test_part