You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by nz...@apache.org on 2010/01/23 02:21:13 UTC
svn commit: r902329 - in /hadoop/hive/trunk: CHANGES.txt
ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
ql/src/test/queries/clientpositive/rcfile_default_format.q
ql/src/test/results/clientpositive/rcfile_default_format.q.out
Author: nzhang
Date: Sat Jan 23 01:21:12 2010
New Revision: 902329
URL: http://svn.apache.org/viewvc?rev=902329&view=rev
Log:
HIVE-1085: ColumnarSerde should not be the default Serde when user specified a fileformat using stored as (Yongqiang He via Ning Zhang)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hadoop/hive/trunk/ql/src/test/queries/clientpositive/rcfile_default_format.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/rcfile_default_format.q.out
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=902329&r1=902328&r2=902329&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Sat Jan 23 01:21:12 2010
@@ -56,6 +56,9 @@
HIVE-1072. Show table extended to error out when the partition does not exist.
(Yongqiang He via zshao)
+ HIVE-1085. ColumnarSerde should not be the default Serde when user
+ specified a fileformat using 'stored as' (Yongqiang He via Ning Zhang)
+
Release 0.5.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=902329&r1=902328&r2=902329&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sat Jan 23 01:21:12 2010
@@ -5884,8 +5884,8 @@
String mapKeyDelim = null;
String lineDelim = null;
String comment = null;
- String inputFormat = TEXTFILE_INPUT;
- String outputFormat = TEXTFILE_OUTPUT;
+ String inputFormat = null;
+ String outputFormat = null;
String location = null;
String serde = null;
Map<String, String> mapProp = null;
@@ -5897,17 +5897,6 @@
final int CTAS = 2; // CREATE TABLE AS SELECT ... (CTAS)
int command_type = CREATE_TABLE;
- if ("SequenceFile".equalsIgnoreCase(conf
- .getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT))) {
- inputFormat = SEQUENCEFILE_INPUT;
- outputFormat = SEQUENCEFILE_OUTPUT;
- } else if ("RCFile".equalsIgnoreCase(conf
- .getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT))) {
- inputFormat = RCFILE_INPUT;
- outputFormat = RCFILE_OUTPUT;
- serde = COLUMNAR_SERDE;
- }
-
LOG.info("Creating table " + tableName + " position="
+ ast.getCharPositionInLine());
int numCh = ast.getChildCount();
@@ -6052,6 +6041,21 @@
}
}
+ if (inputFormat == null) {
+ assert outputFormat == null;
+ if ("SequenceFile".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT))) {
+ inputFormat = SEQUENCEFILE_INPUT;
+ outputFormat = SEQUENCEFILE_OUTPUT;
+ } else if ("RCFile".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT))) {
+ inputFormat = RCFILE_INPUT;
+ outputFormat = RCFILE_OUTPUT;
+ serde = COLUMNAR_SERDE;
+ } else {
+ inputFormat = TEXTFILE_INPUT;
+ outputFormat = TEXTFILE_OUTPUT;
+ }
+ }
+
// check for existence of table
if (ifNotExists) {
try {
Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/rcfile_default_format.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/rcfile_default_format.q?rev=902329&r1=902328&r2=902329&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/rcfile_default_format.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/rcfile_default_format.q Sat Jan 23 01:21:12 2010
@@ -6,6 +6,15 @@
CREATE TABLE rcfile_default_format_ctas AS SELECT key,value FROM src;
DESCRIBE EXTENDED rcfile_default_format_ctas;
+CREATE TABLE rcfile_default_format_txtfile (key STRING) STORED AS TEXTFILE;
+INSERT OVERWRITE TABLE rcfile_default_format_txtfile SELECT key from src;
+DESCRIBE EXTENDED rcfile_default_format_txtfile;
+
SET hive.default.fileformat = TextFile;
CREATE TABLE textfile_default_format_ctas AS SELECT key,value FROM rcfile_default_format_ctas;
-DESCRIBE EXTENDED textfile_default_format_ctas;
\ No newline at end of file
+DESCRIBE EXTENDED textfile_default_format_ctas;
+
+DROP TABLE rcfile_default_format;
+DROP TABLE rcfile_default_format_ctas;
+DROP TABLE rcfile_default_format_txtfile;
+DROP TABLE textfile_default_format_ctas;
\ No newline at end of file
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/rcfile_default_format.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/rcfile_default_format.q.out?rev=902329&r1=902328&r2=902329&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/rcfile_default_format.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/rcfile_default_format.q.out Sat Jan 23 01:21:12 2010
@@ -9,7 +9,7 @@
POSTHOOK: type: DESCTABLE
key string from deserializer
-Detailed Table Information Table(tableName:rcfile_default_format, dbName:default, owner:heyongqiang, createTime:1257212499, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/rcfile_default_format, inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1257212499})
+Detailed Table Information Table(tableName:rcfile_default_format, dbName:default, owner:heyongqiang, createTime:1264196677, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-trunk/build/ql/test/data/warehouse/rcfile_default_format, inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1264196677}, viewOriginalText:null, viewExpandedText:null)
PREHOOK: query: CREATE TABLE rcfile_default_format_ctas AS SELECT key,value FROM src
PREHOOK: type: CREATETABLE
PREHOOK: Input: default@src
@@ -24,7 +24,27 @@
key string from deserializer
value string from deserializer
-Detailed Table Information Table(tableName:rcfile_default_format_ctas, dbName:default, owner:heyongqiang, createTime:1257212511, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:null), FieldSchema(name:value, type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/rcfile_default_format_ctas, inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1257212511})
+Detailed Table Information Table(tableName:rcfile_default_format_ctas, dbName:default, owner:heyongqiang, createTime:1264196683, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:null), FieldSchema(name:value, type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-trunk/build/ql/test/data/warehouse/rcfile_default_format_ctas, inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1264196683}, viewOriginalText:null, viewExpandedText:null)
+PREHOOK: query: CREATE TABLE rcfile_default_format_txtfile (key STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE rcfile_default_format_txtfile (key STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@rcfile_default_format_txtfile
+PREHOOK: query: INSERT OVERWRITE TABLE rcfile_default_format_txtfile SELECT key from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@rcfile_default_format_txtfile
+POSTHOOK: query: INSERT OVERWRITE TABLE rcfile_default_format_txtfile SELECT key from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@rcfile_default_format_txtfile
+PREHOOK: query: DESCRIBE EXTENDED rcfile_default_format_txtfile
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE EXTENDED rcfile_default_format_txtfile
+POSTHOOK: type: DESCTABLE
+key string
+
+Detailed Table Information Table(tableName:rcfile_default_format_txtfile, dbName:default, owner:heyongqiang, createTime:1264196683, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-trunk/build/ql/test/data/warehouse/rcfile_default_format_txtfile, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1264196683}, viewOriginalText:null, viewExpandedText:null)
PREHOOK: query: CREATE TABLE textfile_default_format_ctas AS SELECT key,value FROM rcfile_default_format_ctas
PREHOOK: type: CREATETABLE
PREHOOK: Input: default@rcfile_default_format_ctas
@@ -39,4 +59,24 @@
key string
value string
-Detailed Table Information Table(tableName:textfile_default_format_ctas, dbName:default, owner:heyongqiang, createTime:1257212516, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:null), FieldSchema(name:value, type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/textfile_default_format_ctas, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1257212516})
+Detailed Table Information Table(tableName:textfile_default_format_ctas, dbName:default, owner:heyongqiang, createTime:1264196692, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:null), FieldSchema(name:value, type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-trunk/build/ql/test/data/warehouse/textfile_default_format_ctas, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1264196692}, viewOriginalText:null, viewExpandedText:null)
+PREHOOK: query: DROP TABLE rcfile_default_format
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE rcfile_default_format
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@rcfile_default_format
+PREHOOK: query: DROP TABLE rcfile_default_format_ctas
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE rcfile_default_format_ctas
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@rcfile_default_format_ctas
+PREHOOK: query: DROP TABLE rcfile_default_format_txtfile
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE rcfile_default_format_txtfile
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@rcfile_default_format_txtfile
+PREHOOK: query: DROP TABLE textfile_default_format_ctas
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE textfile_default_format_ctas
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@textfile_default_format_ctas