You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2010/02/20 03:34:54 UTC
svn commit: r912066 - in /hadoop/hive/branches/branch-0.5: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: zshao
Date: Sat Feb 20 02:34:53 2010
New Revision: 912066
URL: http://svn.apache.org/viewvc?rev=912066&view=rev
Log:
HIVE-1181. Make it optional to check file format. (Yongqiang He via zshao)
Added:
hadoop/hive/branches/branch-0.5/ql/src/test/queries/clientpositive/disable_file_format_check.q
hadoop/hive/branches/branch-0.5/ql/src/test/results/clientpositive/disable_file_format_check.q.out
Modified:
hadoop/hive/branches/branch-0.5/CHANGES.txt
hadoop/hive/branches/branch-0.5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hadoop/hive/branches/branch-0.5/conf/hive-default.xml
hadoop/hive/branches/branch-0.5/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
Modified: hadoop/hive/branches/branch-0.5/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.5/CHANGES.txt?rev=912066&r1=912065&r2=912066&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.5/CHANGES.txt (original)
+++ hadoop/hive/branches/branch-0.5/CHANGES.txt Sat Feb 20 02:34:53 2010
@@ -182,6 +182,8 @@
HIVE-1182. Fix TestCliDriver too many open file error.
(Zheng Shao via Ning Zhang)
+ HIVE-1181. Make it optional to check file format. (Yongqiang He via zshao)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/hive/branches/branch-0.5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=912066&r1=912065&r2=912066&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/branches/branch-0.5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Sat Feb 20 02:34:53 2010
@@ -141,6 +141,7 @@
// Default file format for CREATE TABLE statement
// Options: TextFile, SequenceFile
HIVEDEFAULTFILEFORMAT("hive.default.fileformat", "TextFile"),
+ HIVECHECKFILEFORMAT("hive.fileformat.check", true),
//Location of Hive run time structured log file
HIVEHISTORYFILELOC("hive.querylog.location", "/tmp/"+System.getProperty("user.name")),
Modified: hadoop/hive/branches/branch-0.5/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.5/conf/hive-default.xml?rev=912066&r1=912065&r2=912066&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.5/conf/hive-default.xml (original)
+++ hadoop/hive/branches/branch-0.5/conf/hive-default.xml Sat Feb 20 02:34:53 2010
@@ -199,6 +199,12 @@
</property>
<property>
+ <name>hive.fileformat.check</name>
+ <value>true</value>
+ <description>Whether to check file format or not when loading data files</description>
+</property>
+
+<property>
<name>hive.map.aggr</name>
<value>true</value>
<description>Whether to use map-side aggregation in Hive Group By queries</description>
Modified: hadoop/hive/branches/branch-0.5/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.5/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java?rev=912066&r1=912065&r2=912066&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.5/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (original)
+++ hadoop/hive/branches/branch-0.5/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java Sat Feb 20 02:34:53 2010
@@ -28,6 +28,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
@@ -129,10 +130,14 @@
throw new HiveException("addFiles: filesystem error in check phase", e);
}
- // Check if the file format of the file matches that of the table.
- boolean flag = HiveFileFormatUtils.checkInputFormat(fs, conf, tbd.getTable().getInputFileFormatClass(), files);
- if(!flag)
- throw new HiveException("Wrong file format. Please check the file's format.");
+ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVECHECKFILEFORMAT)) {
+ // Check if the file format of the file matches that of the table.
+ boolean flag = HiveFileFormatUtils.checkInputFormat(fs, conf, tbd.getTable().getInputFileFormatClass(), files);
+ if (!flag) {
+ throw new HiveException(
+ "Wrong file format. Please check the file's format.");
+ }
+ }
}
if(tbd.getPartitionSpec().size() == 0) {
Added: hadoop/hive/branches/branch-0.5/ql/src/test/queries/clientpositive/disable_file_format_check.q
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.5/ql/src/test/queries/clientpositive/disable_file_format_check.q?rev=912066&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.5/ql/src/test/queries/clientpositive/disable_file_format_check.q (added)
+++ hadoop/hive/branches/branch-0.5/ql/src/test/queries/clientpositive/disable_file_format_check.q Sat Feb 20 02:34:53 2010
@@ -0,0 +1,9 @@
+set hive.fileformat.check = false;
+create table kv_fileformat_check_txt (key string, value string) stored as textfile;
+load data local inpath '../data/files/kv1.seq' overwrite into table kv_fileformat_check_txt;
+
+create table kv_fileformat_check_seq (key string, value string) stored as sequencefile;
+load data local inpath '../data/files/kv1.txt' overwrite into table kv_fileformat_check_seq;
+
+drop table kv_fileformat_check_seq;
+drop table kv_fileformat_check_txt;
Added: hadoop/hive/branches/branch-0.5/ql/src/test/results/clientpositive/disable_file_format_check.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.5/ql/src/test/results/clientpositive/disable_file_format_check.q.out?rev=912066&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.5/ql/src/test/results/clientpositive/disable_file_format_check.q.out (added)
+++ hadoop/hive/branches/branch-0.5/ql/src/test/results/clientpositive/disable_file_format_check.q.out Sat Feb 20 02:34:53 2010
@@ -0,0 +1,30 @@
+PREHOOK: query: create table kv_fileformat_check_txt (key string, value string) stored as textfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table kv_fileformat_check_txt (key string, value string) stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@kv_fileformat_check_txt
+PREHOOK: query: load data local inpath '../data/files/kv1.seq' overwrite into table kv_fileformat_check_txt
+PREHOOK: type: LOAD
+POSTHOOK: query: load data local inpath '../data/files/kv1.seq' overwrite into table kv_fileformat_check_txt
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@kv_fileformat_check_txt
+PREHOOK: query: create table kv_fileformat_check_seq (key string, value string) stored as sequencefile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table kv_fileformat_check_seq (key string, value string) stored as sequencefile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@kv_fileformat_check_seq
+PREHOOK: query: load data local inpath '../data/files/kv1.txt' overwrite into table kv_fileformat_check_seq
+PREHOOK: type: LOAD
+POSTHOOK: query: load data local inpath '../data/files/kv1.txt' overwrite into table kv_fileformat_check_seq
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@kv_fileformat_check_seq
+PREHOOK: query: drop table kv_fileformat_check_seq
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table kv_fileformat_check_seq
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@kv_fileformat_check_seq
+PREHOOK: query: drop table kv_fileformat_check_txt
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table kv_fileformat_check_txt
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@kv_fileformat_check_txt