You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2013/12/16 16:03:45 UTC
svn commit: r1551221 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
test/queries/clientpositive/loadpart2.q
test/results/clientnegative/dyn_part4.q.out
test/results/clientpositive/loadpart2.q.out
Author: xuefu
Date: Mon Dec 16 15:03:45 2013
New Revision: 1551221
URL: http://svn.apache.org/r1551221
Log:
HIVE-6022: Load statements with incorrect order of partitions put input files to unreadable places (Teruyoshi Zenmyo via Xuefu)
Added:
hive/trunk/ql/src/test/queries/clientpositive/loadpart2.q
hive/trunk/ql/src/test/results/clientpositive/loadpart2.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1551221&r1=1551220&r2=1551221&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Mon Dec 16 15:03:45 2013
@@ -750,7 +750,7 @@ public abstract class BaseSemanticAnalyz
ASTNode partspec = (ASTNode) ast.getChild(1);
partitions = new ArrayList<Partition>();
// partSpec is a mapping from partition column name to its value.
- partSpec = new LinkedHashMap<String, String>(partspec.getChildCount());
+ Map<String, String> tmpPartSpec = new HashMap<String, String>(partspec.getChildCount());
for (int i = 0; i < partspec.getChildCount(); ++i) {
ASTNode partspec_val = (ASTNode) partspec.getChild(i);
String val = null;
@@ -765,15 +765,21 @@ public abstract class BaseSemanticAnalyz
} else { // in the form of T partition (ds="2010-03-03")
val = stripQuotes(partspec_val.getChild(1).getText());
}
- partSpec.put(colName, val);
+ tmpPartSpec.put(colName, val);
}
// check if the columns, as well as value types in the partition() clause are valid
- validatePartSpec(tableHandle, partSpec, ast, conf);
+ validatePartSpec(tableHandle, tmpPartSpec, ast, conf);
+
+ List<FieldSchema> parts = tableHandle.getPartitionKeys();
+ partSpec = new LinkedHashMap<String, String>(partspec.getChildCount());
+ for (FieldSchema fs : parts) {
+ String partKey = fs.getName();
+ partSpec.put(partKey, tmpPartSpec.get(partKey));
+ }
// check if the partition spec is valid
if (numDynParts > 0) {
- List<FieldSchema> parts = tableHandle.getPartitionKeys();
int numStaPart = parts.size() - numDynParts;
if (numStaPart == 0 &&
conf.getVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equalsIgnoreCase("strict")) {
Added: hive/trunk/ql/src/test/queries/clientpositive/loadpart2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/loadpart2.q?rev=1551221&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/loadpart2.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/loadpart2.q Mon Dec 16 15:03:45 2013
@@ -0,0 +1,9 @@
+
+create table hive_test ( col1 string ) partitioned by ( pcol1 string , pcol2 string) stored as textfile;
+load data local inpath '../../data/files/test.dat' overwrite into table hive_test partition (pcol1='part1',pcol2='part1') ;
+load data local inpath '../../data/files/test.dat' overwrite into table hive_test partition (pcol2='part2',pcol1='part2') ;
+select * from hive_test where pcol1='part1' and pcol2='part1';
+select * from hive_test where pcol1='part2' and pcol2='part2';
+
+
+
Modified: hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out?rev=1551221&r1=1551220&r2=1551221&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out Mon Dec 16 15:03:45 2013
@@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE
POSTHOOK: query: create table nzhang_part4 (key string) partitioned by (ds string, hr string, value string)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@nzhang_part4
-FAILED: SemanticException [Error 10125]: Partition columns in partition specification are not the same as that defined in the table schema. The names and orders have to be exactly the same. Partition columns in the table schema are: (ds, hr, value), while the partitions specified in the query are: (value, ds, hr).
+FAILED: SemanticException [Error 10094]: Line 3:46 Dynamic partition cannot be the parent of a static partition 'hr'
Added: hive/trunk/ql/src/test/results/clientpositive/loadpart2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/loadpart2.q.out?rev=1551221&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/loadpart2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/loadpart2.q.out Mon Dec 16 15:03:45 2013
@@ -0,0 +1,51 @@
+PREHOOK: query: create table hive_test ( col1 string ) partitioned by ( pcol1 string , pcol2 string) stored as textfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table hive_test ( col1 string ) partitioned by ( pcol1 string , pcol2 string) stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@hive_test
+PREHOOK: query: load data local inpath '../../data/files/test.dat' overwrite into table hive_test partition (pcol1='part1',pcol2='part1')
+PREHOOK: type: LOAD
+PREHOOK: Output: default@hive_test
+POSTHOOK: query: load data local inpath '../../data/files/test.dat' overwrite into table hive_test partition (pcol1='part1',pcol2='part1')
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@hive_test
+POSTHOOK: Output: default@hive_test@pcol1=part1/pcol2=part1
+PREHOOK: query: load data local inpath '../../data/files/test.dat' overwrite into table hive_test partition (pcol2='part2',pcol1='part2')
+PREHOOK: type: LOAD
+PREHOOK: Output: default@hive_test
+POSTHOOK: query: load data local inpath '../../data/files/test.dat' overwrite into table hive_test partition (pcol2='part2',pcol1='part2')
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@hive_test
+POSTHOOK: Output: default@hive_test@pcol1=part2/pcol2=part2
+PREHOOK: query: select * from hive_test where pcol1='part1' and pcol2='part1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hive_test
+PREHOOK: Input: default@hive_test@pcol1=part1/pcol2=part1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hive_test where pcol1='part1' and pcol2='part1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hive_test
+POSTHOOK: Input: default@hive_test@pcol1=part1/pcol2=part1
+#### A masked pattern was here ####
+1 part1 part1
+2 part1 part1
+3 part1 part1
+4 part1 part1
+5 part1 part1
+6 part1 part1
+PREHOOK: query: select * from hive_test where pcol1='part2' and pcol2='part2'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hive_test
+PREHOOK: Input: default@hive_test@pcol1=part2/pcol2=part2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hive_test where pcol1='part2' and pcol2='part2'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hive_test
+POSTHOOK: Input: default@hive_test@pcol1=part2/pcol2=part2
+#### A masked pattern was here ####
+1 part2 part2
+2 part2 part2
+3 part2 part2
+4 part2 part2
+5 part2 part2
+6 part2 part2