You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2009/04/09 08:05:21 UTC
svn commit: r763521 - in /hadoop/hive/branches/branch-0.3: ./
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/parse/
ql/src/java/org/apache/hadoop/hive/ql/plan/
Author: namit
Date: Thu Apr 9 06:05:21 2009
New Revision: 763521
URL: http://svn.apache.org/viewvc?rev=763521&view=rev
Log:
HIVE-393. Remove unnecessary checks for file type.
(Zheng Shao via namit)
Modified:
hadoop/hive/branches/branch-0.3/CHANGES.txt
hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/plan/moveWork.java
Modified: hadoop/hive/branches/branch-0.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.3/CHANGES.txt?rev=763521&r1=763520&r2=763521&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.3/CHANGES.txt (original)
+++ hadoop/hive/branches/branch-0.3/CHANGES.txt Thu Apr 9 06:05:21 2009
@@ -22,6 +22,9 @@
HIVE-299. Include php packages for thrift service.
(Raghotham Murthy via zshao)
+ HIVE-393. Remove unnecessary checks for file type.
+ (Zheng Shao via namit)
+
IMPROVEMENTS
HIVE-132. Show table and describe results to be read via FetchTask.
Modified: hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java?rev=763521&r1=763520&r2=763521&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (original)
+++ hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java Thu Apr 9 06:05:21 2009
@@ -99,43 +99,44 @@
String mesg_detail = " from " + tbd.getSourceDir();
console.printInfo(mesg, mesg_detail);
- // Get the file format of the table
- boolean tableIsSequenceFile = tbd.getTable().getInputFileFormatClass().equals(SequenceFileInputFormat.class);
- // Get all files from the src directory
- FileStatus [] dirs;
- ArrayList<FileStatus> files;
- try {
- fs = FileSystem.get(db.getTable(tbd.getTable().getTableName()).getDataLocation(),
- Hive.get().getConf());
- dirs = fs.globStatus(new Path(tbd.getSourceDir()));
- files = new ArrayList<FileStatus>();
- for (int i=0; (dirs != null && i<dirs.length); i++) {
- files.addAll(Arrays.asList(fs.listStatus(dirs[i].getPath())));
- // We only check one file, so exit the loop when we have at least one.
- if (files.size()>0) break;
- }
- } catch (IOException e) {
- throw new HiveException("addFiles: filesystem error in check phase", e);
- }
- // Check if the file format of the file matches that of the table.
- if (files.size() > 0) {
- int fileId = 0;
- boolean fileIsSequenceFile = true;
+ if (work.getCheckFileFormat()) {
+ // Get the file format of the table
+ boolean tableIsSequenceFile = tbd.getTable().getInputFileFormatClass().equals(SequenceFileInputFormat.class);
+ // Get all files from the src directory
+ FileStatus [] dirs;
+ ArrayList<FileStatus> files;
try {
- SequenceFile.Reader reader = new SequenceFile.Reader(
- fs, files.get(fileId).getPath(), conf);
- reader.close();
+ fs = FileSystem.get(db.getTable(tbd.getTable().getTableName()).getDataLocation(),
+ Hive.get().getConf());
+ dirs = fs.globStatus(new Path(tbd.getSourceDir()));
+ files = new ArrayList<FileStatus>();
+ for (int i=0; (dirs != null && i<dirs.length); i++) {
+ files.addAll(Arrays.asList(fs.listStatus(dirs[i].getPath())));
+ // We only check one file, so exit the loop when we have at least one.
+ if (files.size()>0) break;
+ }
} catch (IOException e) {
- fileIsSequenceFile = false;
+ throw new HiveException("addFiles: filesystem error in check phase", e);
}
- if (!fileIsSequenceFile && tableIsSequenceFile) {
- throw new HiveException("Cannot load text files into a table stored as SequenceFile.");
+ // Check if the file format of the file matches that of the table.
+ if (files.size() > 0) {
+ int fileId = 0;
+ boolean fileIsSequenceFile = true;
+ try {
+ SequenceFile.Reader reader = new SequenceFile.Reader(
+ fs, files.get(fileId).getPath(), conf);
+ reader.close();
+ } catch (IOException e) {
+ fileIsSequenceFile = false;
+ }
+ if (!fileIsSequenceFile && tableIsSequenceFile) {
+ throw new HiveException("Cannot load text files into a table stored as SequenceFile.");
+ }
+ if (fileIsSequenceFile && !tableIsSequenceFile) {
+ throw new HiveException("Cannot load SequenceFiles into a table stored as TextFile.");
+ }
}
- if (fileIsSequenceFile && !tableIsSequenceFile) {
- throw new HiveException("Cannot load SequenceFiles into a table stored as TextFile.");
- }
- }
-
+ }
if(tbd.getPartitionSpec().size() == 0) {
db.loadTable(new Path(tbd.getSourceDir()), tbd.getTable().getTableName(), tbd.getReplace(), new Path(tbd.getTmpDir()));
Modified: hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java?rev=763521&r1=763520&r2=763521&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (original)
+++ hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java Thu Apr 9 06:05:21 2009
@@ -213,9 +213,9 @@
isOverWrite));
if(rTask != null) {
- rTask.addDependentTask(TaskFactory.get(new moveWork(loadTableWork, loadFileWork), this.conf));
+ rTask.addDependentTask(TaskFactory.get(new moveWork(loadTableWork, loadFileWork, true), this.conf));
} else {
- rTask = TaskFactory.get(new moveWork(loadTableWork, loadFileWork), this.conf);
+ rTask = TaskFactory.get(new moveWork(loadTableWork, loadFileWork, true), this.conf);
}
rootTasks.add(rTask);
Modified: hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=763521&r1=763520&r2=763521&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu Apr 9 06:05:21 2009
@@ -3337,7 +3337,7 @@
else {
// First we generate the move work as this needs to be made dependent on all
// the tasks that have a file sink operation
- mv = new moveWork(loadTableWork, loadFileWork);
+ mv = new moveWork(loadTableWork, loadFileWork, false);
mvTask = TaskFactory.get(mv, this.conf);
}
Modified: hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/plan/moveWork.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/plan/moveWork.java?rev=763521&r1=763520&r2=763521&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/plan/moveWork.java (original)
+++ hadoop/hive/branches/branch-0.3/ql/src/java/org/apache/hadoop/hive/ql/plan/moveWork.java Thu Apr 9 06:05:21 2009
@@ -27,12 +27,16 @@
private List<loadTableDesc> loadTableWork;
private List<loadFileDesc> loadFileWork;
+ private boolean checkFileFormat;
+
public moveWork() { }
public moveWork(
final List<loadTableDesc> loadTableWork,
- final List<loadFileDesc> loadFileWork) {
+ final List<loadFileDesc> loadFileWork,
+ boolean checkFileFormat) {
this.loadTableWork = loadTableWork;
this.loadFileWork = loadFileWork;
+ this.checkFileFormat = checkFileFormat;
}
@explain(displayName="tables")
public List<loadTableDesc> getLoadTableWork() {
@@ -49,4 +53,12 @@
public void setLoadFileWork(final List<loadFileDesc> loadFileWork) {
this.loadFileWork=loadFileWork;
}
+
+ public boolean getCheckFileFormat() {
+ return checkFileFormat;
+ }
+ public void setCheckFileFormat(boolean checkFileFormat) {
+ this.checkFileFormat = checkFileFormat;
+ }
+
}