svn commit: r1542424 - /hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java

[gu...@apache.org]

Author: gunther
Date: Fri Nov 15 22:51:31 2013
New Revision: 1542424

URL: http://svn.apache.org/r1542424
Log:
HIVE-5834: Avoid reading ORC footers for files which will not be split in OrcInputFormat::getSplits() (Gopal V via Gunther Hagleitner)

Modified:
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java?rev=1542424&r1=1542423&r2=1542424&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java Fri Nov 15 22:51:31 2013
@@ -475,7 +475,8 @@ public class OrcInputFormat  implements 
             if (context.cacheStripeDetails) {
               fileInfo = verifyCachedFileInfo(file);
             }
-            context.schedule(new SplitGenerator(context, fs, file, fileInfo));
+            SplitGenerator spgen = new SplitGenerator(context, fs, file, fileInfo);
+            spgen.schedule();
           }
         }
       } catch (Throwable th) {
@@ -553,6 +554,19 @@ public class OrcInputFormat  implements 
       return file.getPath();
     }
 
+    void schedule() throws IOException {
+      if(locations.length == 1 && file.getLen() < context.maxSize) {
+        String[] hosts = locations[0].getHosts();
+        synchronized (context.splits) {
+          context.splits.add(new OrcSplit(file.getPath(), 0, file.getLen(),
+                hosts, fileMetaInfo));
+        }
+      } else {
+        // if it requires a compute task
+        context.schedule(this);
+      }
+    }
+
     @Override
     public String toString() {
       return "splitter(" + file.getPath() + ")";
@@ -877,4 +891,4 @@ public class OrcInputFormat  implements 
       this.types = types;
     }
   }
-}
\ No newline at end of file
+}