You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by he...@apache.org on 2010/05/21 08:39:53 UTC

svn commit: r946902 - in /hadoop/hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/metadata/ ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/

Author: heyongqiang
Date: Fri May 21 06:39:53 2010
New Revision: 946902

URL: http://svn.apache.org/viewvc?rev=946902&view=rev
Log:
HIVE-1354. partition level properties honored if it exists (namit via He Yongqiang)

Added:
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat3.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=946902&r1=946901&r2=946902&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri May 21 06:39:53 2010
@@ -447,6 +447,9 @@ Trunk -  Unreleased
     HIVE-1353. load_dyn_part*.q need ORDER BY for determinism
     (John Sichi via Ning Zhang)
 
+    HIVE-1354. partition level properties honored if it exists
+    (namit via He Yongqiang)
+
 Release 0.5.0 -  Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=946902&r1=946901&r2=946902&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Fri May 21 06:39:53 2010
@@ -567,16 +567,15 @@ public class Hive {
         partPath = part.getPath()[0];
         fs = partPath.getFileSystem(getConf());
       }
+
       if (replace) {
         Hive.replaceFiles(loadPath, partPath, fs, tmpDirPath);
       } else {
         Hive.copyFiles(loadPath, partPath, fs);
       }
 
-      if (part == null) {
-        // create the partition if it didn't exist before
-        part = getPartition(tbl, partSpec, true);
-      }
+      // recreate the partition if it existed before
+      part = getPartition(tbl, partSpec, true);
     } catch (IOException e) {
       LOG.error(StringUtils.stringifyException(e));
       throw new HiveException(e);
@@ -755,10 +754,21 @@ public class Hive {
     org.apache.hadoop.hive.metastore.api.Partition tpart = null;
     try {
       tpart = getMSC().getPartition(tbl.getDbName(), tbl.getTableName(), pvals);
-      if (tpart == null && forceCreate) {
-        LOG.debug("creating partition for table " + tbl.getTableName()
-            + " with partition spec : " + partSpec);
-        tpart = getMSC().appendPartition(tbl.getDbName(), tbl.getTableName(), pvals);
+      if (forceCreate) {
+        if (tpart == null) {
+          LOG.debug("creating partition for table " + tbl.getTableName()
+                    + " with partition spec : " + partSpec);
+          tpart = getMSC().appendPartition(tbl.getDbName(), tbl.getTableName(), pvals);
+        }
+        else {
+          LOG.debug("altering partition for table " + tbl.getTableName()
+                    + " with partition spec : " + partSpec);
+
+          tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat());
+          tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat());
+          tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib());
+          alterPartition(tbl.getTableName(), new Partition(tbl, tpart));
+        }
       }
       if (tpart == null) {
         return null;

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=946902&r1=946901&r2=946902&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Fri May 21 06:39:53 2010
@@ -504,13 +504,8 @@ public abstract class BaseSemanticAnalyz
           partHandle = null;
         } else {
           try {
-            // In case the partition already exists, we need to get the partition
-          	// data from the metastore
-            partHandle = db.getPartition(tableHandle, partSpec, false);
-	          if (partHandle == null) {
-	            // this doesn't create partition. partition is created in MoveTask
-  	          partHandle = new Partition(tableHandle, partSpec, null);
-	          }
+            // this doesn't create partition. partition is created in MoveTask
+            partHandle = new Partition(tableHandle, partSpec, null);
         	} catch (HiveException e) {
          		throw new SemanticException(
          		    ErrorMsg.INVALID_PARTITION.getMsg(ast.getChild(childIndex)));

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat3.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat3.q?rev=946902&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat3.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat3.q Fri May 21 06:39:53 2010
@@ -0,0 +1,18 @@
+drop table partition_test_partitioned;
+
+create table partition_test_partitioned(key string, value string) partitioned by (dt string);
+
+alter table partition_test_partitioned set fileformat rcfile;
+insert overwrite table partition_test_partitioned partition(dt=101) select * from src1;
+show table extended like partition_test_partitioned partition(dt=101);
+
+alter table partition_test_partitioned set fileformat Sequencefile;
+insert overwrite table partition_test_partitioned partition(dt=102) select * from src1;
+show table extended like partition_test_partitioned partition(dt=102);
+select key from partition_test_partitioned where dt=102;
+
+insert overwrite table partition_test_partitioned partition(dt=101) select * from src1;
+show table extended like partition_test_partitioned partition(dt=101);
+select key from partition_test_partitioned where dt=101;
+
+drop table partition_test_partitioned;

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out?rev=946902&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out Fri May 21 06:39:53 2010
@@ -0,0 +1,215 @@
+PREHOOK: query: drop table partition_test_partitioned
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table partition_test_partitioned
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table partition_test_partitioned(key string, value string) partitioned by (dt string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table partition_test_partitioned(key string, value string) partitioned by (dt string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@partition_test_partitioned
+PREHOOK: query: alter table partition_test_partitioned set fileformat rcfile
+PREHOOK: type: null
+POSTHOOK: query: alter table partition_test_partitioned set fileformat rcfile
+POSTHOOK: type: null
+POSTHOOK: Input: default@partition_test_partitioned
+POSTHOOK: Output: default@partition_test_partitioned
+PREHOOK: query: insert overwrite table partition_test_partitioned partition(dt=101) select * from src1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src1
+PREHOOK: Output: default@partition_test_partitioned@dt=101
+POSTHOOK: query: insert overwrite table partition_test_partitioned partition(dt=101) select * from src1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src1
+POSTHOOK: Output: default@partition_test_partitioned@dt=101
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: show table extended like partition_test_partitioned partition(dt=101)
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like partition_test_partitioned partition(dt=101)
+POSTHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+tableName:partition_test_partitioned
+owner:njain
+location:file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/partition_test_partitioned/dt=101
+inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+columns:struct columns { string key, string value}
+partitioned:true
+partitionColumns:struct partition_columns { string dt}
+totalNumberFiles:1
+totalFileSize:370
+maxFileSize:370
+minFileSize:370
+lastAccessTime:0
+lastUpdateTime:1274314890000
+
+PREHOOK: query: alter table partition_test_partitioned set fileformat Sequencefile
+PREHOOK: type: null
+POSTHOOK: query: alter table partition_test_partitioned set fileformat Sequencefile
+POSTHOOK: type: null
+POSTHOOK: Input: default@partition_test_partitioned
+POSTHOOK: Output: default@partition_test_partitioned
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table partition_test_partitioned partition(dt=102) select * from src1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src1
+PREHOOK: Output: default@partition_test_partitioned@dt=102
+POSTHOOK: query: insert overwrite table partition_test_partitioned partition(dt=102) select * from src1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src1
+POSTHOOK: Output: default@partition_test_partitioned@dt=102
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: show table extended like partition_test_partitioned partition(dt=102)
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like partition_test_partitioned partition(dt=102)
+POSTHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+tableName:partition_test_partitioned
+owner:njain
+location:file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/partition_test_partitioned/dt=102
+inputformat:org.apache.hadoop.mapred.SequenceFileInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+columns:struct columns { string key, string value}
+partitioned:true
+partitionColumns:struct partition_columns { string dt}
+totalNumberFiles:1
+totalFileSize:888
+maxFileSize:888
+minFileSize:888
+lastAccessTime:0
+lastUpdateTime:1274314895000
+
+PREHOOK: query: select key from partition_test_partitioned where dt=102
+PREHOOK: type: QUERY
+PREHOOK: Input: default@partition_test_partitioned@dt=102
+PREHOOK: Output: file:/data/users/njain/hive3/hive3/build/ql/scratchdir/hive_2010-05-19_17-21-36_438_6571635211032352917/10000
+POSTHOOK: query: select key from partition_test_partitioned where dt=102
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@partition_test_partitioned@dt=102
+POSTHOOK: Output: file:/data/users/njain/hive3/hive3/build/ql/scratchdir/hive_2010-05-19_17-21-36_438_6571635211032352917/10000
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+238
+
+311
+
+
+
+255
+278
+98
+
+
+
+401
+150
+273
+224
+369
+66
+128
+213
+146
+406
+
+
+
+PREHOOK: query: insert overwrite table partition_test_partitioned partition(dt=101) select * from src1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src1
+PREHOOK: Output: default@partition_test_partitioned@dt=101
+POSTHOOK: query: insert overwrite table partition_test_partitioned partition(dt=101) select * from src1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src1
+POSTHOOK: Output: default@partition_test_partitioned@dt=101
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: show table extended like partition_test_partitioned partition(dt=101)
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like partition_test_partitioned partition(dt=101)
+POSTHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+tableName:partition_test_partitioned
+owner:njain
+location:file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/partition_test_partitioned/dt=101
+inputformat:org.apache.hadoop.mapred.SequenceFileInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+columns:struct columns { string key, string value}
+partitioned:true
+partitionColumns:struct partition_columns { string dt}
+totalNumberFiles:1
+totalFileSize:888
+maxFileSize:888
+minFileSize:888
+lastAccessTime:0
+lastUpdateTime:1274314904000
+
+PREHOOK: query: select key from partition_test_partitioned where dt=101
+PREHOOK: type: QUERY
+PREHOOK: Input: default@partition_test_partitioned@dt=101
+PREHOOK: Output: file:/data/users/njain/hive3/hive3/build/ql/scratchdir/hive_2010-05-19_17-21-45_060_1172142624101768381/10000
+POSTHOOK: query: select key from partition_test_partitioned where dt=101
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@partition_test_partitioned@dt=101
+POSTHOOK: Output: file:/data/users/njain/hive3/hive3/build/ql/scratchdir/hive_2010-05-19_17-21-45_060_1172142624101768381/10000
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+238
+
+311
+
+
+
+255
+278
+98
+
+
+
+401
+150
+273
+224
+369
+66
+128
+213
+146
+406
+
+
+
+PREHOOK: query: drop table partition_test_partitioned
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table partition_test_partitioned
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@partition_test_partitioned
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]