You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2011/10/25 02:02:04 UTC
svn commit: r1188450 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
test/queries/clientnegative/dyn_part4.q
test/results/clientnegative/dyn_part4.q.out
Author: namit
Date: Tue Oct 25 00:02:04 2011
New Revision: 1188450
URL: http://svn.apache.org/viewvc?rev=1188450&view=rev
Log:
HIVE-2519 Dynamic partition insert should enforce the order of the
partition spec is the same as the one in schema (Ning Zhang via namit)
Added:
hive/trunk/ql/src/test/queries/clientnegative/dyn_part4.q
hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1188450&r1=1188449&r2=1188450&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Tue Oct 25 00:02:04 2011
@@ -23,6 +23,7 @@ import java.io.UnsupportedEncodingExcept
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -675,17 +676,30 @@ public abstract class BaseSemanticAnalyz
conf.getVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equalsIgnoreCase("strict")) {
throw new SemanticException(ErrorMsg.DYNAMIC_PARTITION_STRICT_MODE.getMsg());
}
- for (FieldSchema fs: parts) {
- if (partSpec.get(fs.getName().toLowerCase()) == null) {
- if (numStaPart > 0) { // found a DP, but there exists ST as subpartition
- throw new SemanticException(
- ErrorMsg.PARTITION_DYN_STA_ORDER.getMsg(ast.getChild(childIndex)));
- }
- break;
- } else {
- --numStaPart;
- }
- }
+
+ // check the partitions in partSpec be the same as defined in table schema
+ if (partSpec.keySet().size() != parts.size()) {
+ ErrorPartSpec(partSpec, parts);
+ }
+ Iterator<String> itrPsKeys = partSpec.keySet().iterator();
+ for (FieldSchema fs: parts) {
+ if (!itrPsKeys.next().toLowerCase().equals(fs.getName().toLowerCase())) {
+ ErrorPartSpec(partSpec, parts);
+ }
+ }
+
+ // check if static partition appear after dynamic partitions
+ for (FieldSchema fs: parts) {
+ if (partSpec.get(fs.getName().toLowerCase()) == null) {
+ if (numStaPart > 0) { // found a DP, but there exists ST as subpartition
+ throw new SemanticException(
+ ErrorMsg.PARTITION_DYN_STA_ORDER.getMsg(ast.getChild(childIndex)));
+ }
+ break;
+ } else {
+ --numStaPart;
+ }
+ }
partHandle = null;
specType = SpecType.DYNAMIC_PARTITION;
} else {
@@ -714,6 +728,26 @@ public abstract class BaseSemanticAnalyz
}
}
+ private void ErrorPartSpec(Map<String, String> partSpec, List<FieldSchema> parts)
+ throws SemanticException {
+ StringBuilder sb = new StringBuilder("Partition columns in the table schema are: (");
+ for (FieldSchema fs: parts) {
+ sb.append(fs.getName()).append(", ");
+ }
+ sb.setLength(sb.length() - 2); // remove the last ", "
+ sb.append("), while the partitions specified in the query are: (");
+
+ Iterator<String> itrPsKeys = partSpec.keySet().iterator();
+ while (itrPsKeys.hasNext()) {
+ sb.append(itrPsKeys.next()).append(", ");
+ }
+ sb.setLength(sb.length() - 2); // remove the last ", "
+ sb.append(").");
+
+ throw new SemanticException(
+ ErrorMsg.PARTSPEC_DIFFER_FROM_SCHEMA.getMsg(sb.toString()));
+ }
+
public Map<String, String> getPartSpec() {
return this.partSpec;
}
@@ -730,6 +764,7 @@ public abstract class BaseSemanticAnalyz
return tableHandle.toString();
}
}
+
}
/**
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java?rev=1188450&r1=1188449&r2=1188450&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java Tue Oct 25 00:02:04 2011
@@ -186,7 +186,9 @@ public enum ErrorMsg {
NO_COMPARE_BIGINT_DOUBLE("In strict mode, comparing bigints and doubles is not allowed, "
+ "it may result in a loss of precision. "
+ "If you really want to perform the operation, set hive.mapred.mode=nonstrict"),
- FUNCTIONS_ARE_NOT_SUPPORTED_IN_ORDER_BY("functions are not supported in order by"),
+ FUNCTIONS_ARE_NOT_SUPPORTED_IN_ORDER_BY("functions are not supported in order by"),
+ PARTSPEC_DIFFER_FROM_SCHEMA("Partition columns in partition specification are not the same as "
+ + "that defined in the table schema. The names and orders have to be exactly the same."),
;
private String mesg;
Added: hive/trunk/ql/src/test/queries/clientnegative/dyn_part4.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/dyn_part4.q?rev=1188450&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/dyn_part4.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/dyn_part4.q Tue Oct 25 00:02:04 2011
@@ -0,0 +1,7 @@
+create table nzhang_part4 (key string) partitioned by (ds string, hr string, value string);
+
+set hive.exec.dynamic.partition=true;
+
+insert overwrite table nzhang_part4 partition(value = 'aaa', ds='11', hr) select key, hr from srcpart where ds is not null;
+
+drop table nzhang_part4;
Added: hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out?rev=1188450&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out Tue Oct 25 00:02:04 2011
@@ -0,0 +1,6 @@
+PREHOOK: query: create table nzhang_part4 (key string) partitioned by (ds string, hr string, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table nzhang_part4 (key string) partitioned by (ds string, hr string, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@nzhang_part4
+FAILED: Error in semantic analysis: Partition columns in partition specification are not the same as that defined in the table schema. The names and orders have to be exactly the same. Partition columns in the table schema are: (ds, hr, value), while the partitions specified in the query are: (value, ds, hr).