You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2010/02/20 03:36:29 UTC
svn commit: r912067 - in /hadoop/hive/trunk: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: zshao
Date: Sat Feb 20 02:36:28 2010
New Revision: 912067
URL: http://svn.apache.org/viewvc?rev=912067&view=rev
Log:
HIVE-1181. Make it optional to check file format. (Yongqiang He via zshao)
Added:
hadoop/hive/trunk/ql/src/test/queries/clientpositive/disable_file_format_check.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/disable_file_format_check.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hadoop/hive/trunk/conf/hive-default.xml
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=912067&r1=912066&r2=912067&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Sat Feb 20 02:36:28 2010
@@ -93,6 +93,8 @@
HIVE-1136. Add type-checking setters for HiveConf class.
(John Sichi via zshao)
+ HIVE-1181. Make it optional to check file format. (Yongqiang He via zshao)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=912067&r1=912066&r2=912067&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Sat Feb 20 02:36:28 2010
@@ -147,6 +147,7 @@
// Default file format for CREATE TABLE statement
// Options: TextFile, SequenceFile
HIVEDEFAULTFILEFORMAT("hive.default.fileformat", "TextFile"),
+ HIVECHECKFILEFORMAT("hive.fileformat.check", true),
//Location of Hive run time structured log file
HIVEHISTORYFILELOC("hive.querylog.location", "/tmp/" + System.getProperty("user.name")),
Modified: hadoop/hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/conf/hive-default.xml?rev=912067&r1=912066&r2=912067&view=diff
==============================================================================
--- hadoop/hive/trunk/conf/hive-default.xml (original)
+++ hadoop/hive/trunk/conf/hive-default.xml Sat Feb 20 02:36:28 2010
@@ -199,6 +199,12 @@
</property>
<property>
+ <name>hive.fileformat.check</name>
+ <value>true</value>
+ <description>Whether to check file format or not when loading data files</description>
+</property>
+
+<property>
<name>hive.map.aggr</name>
<value>true</value>
<description>Whether to use map-side aggregation in Hive Group By queries</description>
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java?rev=912067&r1=912066&r2=912067&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java Sat Feb 20 02:36:28 2010
@@ -28,6 +28,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
@@ -140,13 +141,13 @@
throw new HiveException(
"addFiles: filesystem error in check phase", e);
}
-
- // Check if the file format of the file matches that of the table.
- boolean flag = HiveFileFormatUtils.checkInputFormat(fs, conf, tbd
- .getTable().getInputFileFormatClass(), files);
- if (!flag) {
- throw new HiveException(
- "Wrong file format. Please check the file's format.");
+ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVECHECKFILEFORMAT)) {
+ // Check if the file format of the file matches that of the table.
+ boolean flag = HiveFileFormatUtils.checkInputFormat(fs, conf, tbd.getTable().getInputFileFormatClass(), files);
+ if (!flag) {
+ throw new HiveException(
+ "Wrong file format. Please check the file's format.");
+ }
}
}
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/disable_file_format_check.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/disable_file_format_check.q?rev=912067&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/disable_file_format_check.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/disable_file_format_check.q Sat Feb 20 02:36:28 2010
@@ -0,0 +1,9 @@
+set hive.fileformat.check = false;
+create table kv_fileformat_check_txt (key string, value string) stored as textfile;
+load data local inpath '../data/files/kv1.seq' overwrite into table kv_fileformat_check_txt;
+
+create table kv_fileformat_check_seq (key string, value string) stored as sequencefile;
+load data local inpath '../data/files/kv1.txt' overwrite into table kv_fileformat_check_seq;
+
+drop table kv_fileformat_check_seq;
+drop table kv_fileformat_check_txt;
Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/disable_file_format_check.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/disable_file_format_check.q.out?rev=912067&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/disable_file_format_check.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/disable_file_format_check.q.out Sat Feb 20 02:36:28 2010
@@ -0,0 +1,30 @@
+PREHOOK: query: create table kv_fileformat_check_txt (key string, value string) stored as textfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table kv_fileformat_check_txt (key string, value string) stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@kv_fileformat_check_txt
+PREHOOK: query: load data local inpath '../data/files/kv1.seq' overwrite into table kv_fileformat_check_txt
+PREHOOK: type: LOAD
+POSTHOOK: query: load data local inpath '../data/files/kv1.seq' overwrite into table kv_fileformat_check_txt
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@kv_fileformat_check_txt
+PREHOOK: query: create table kv_fileformat_check_seq (key string, value string) stored as sequencefile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table kv_fileformat_check_seq (key string, value string) stored as sequencefile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@kv_fileformat_check_seq
+PREHOOK: query: load data local inpath '../data/files/kv1.txt' overwrite into table kv_fileformat_check_seq
+PREHOOK: type: LOAD
+POSTHOOK: query: load data local inpath '../data/files/kv1.txt' overwrite into table kv_fileformat_check_seq
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@kv_fileformat_check_seq
+PREHOOK: query: drop table kv_fileformat_check_seq
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table kv_fileformat_check_seq
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@kv_fileformat_check_seq
+PREHOOK: query: drop table kv_fileformat_check_txt
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table kv_fileformat_check_txt
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@kv_fileformat_check_txt