You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/11/09 12:44:38 UTC

svn commit: r1407429 - /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java

Author: namit
Date: Fri Nov  9 11:44:37 2012
New Revision: 1407429

URL: http://svn.apache.org/viewvc?rev=1407429&view=rev
Log:
HIVE-3696 Revert HIVE-3483 which causes performance regression
(Gang Tim Liu via namit)


Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java?rev=1407429&r1=1407428&r2=1407429&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java Fri Nov  9 11:44:37 2012
@@ -33,7 +33,6 @@ import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -367,7 +366,7 @@ public class CombineHiveInputFormat<K ex
 
       if (!done) {
         if (f == null) {
-          f = new CombineFilter(job, filterPath);
+          f = new CombineFilter(filterPath);
           LOG.info("CombineHiveInputSplit creating pool for " + path +
                    "; using filter path " + filterPath);
           combine.createPool(job, f);
@@ -394,7 +393,7 @@ public class CombineHiveInputFormat<K ex
       if (inpFiles.size() > 0) {
         // Processing files
         for (Path filterPath : poolSet) {
-          combine.createPool(job, new CombineFilter(job, filterPath));
+          combine.createPool(job, new CombineFilter(filterPath));
         }
         processPaths(job, combine, iss, inpFiles.toArray(new Path[0]));
       }
@@ -545,13 +544,11 @@ public class CombineHiveInputFormat<K ex
   }
 
   static class CombineFilter implements PathFilter {
-    private final Set<Path> filterPaths = new HashSet<Path>();
-    private final Configuration conf;
+    private final Set<String> pStrings = new HashSet<String>();
 
     // store a path prefix in this TestFilter
     // PRECONDITION: p should always be a directory
-    public CombineFilter(Configuration conf, Path p) {
-      this.conf = conf;
+    public CombineFilter(Path p) {
       // we need to keep the path part only because the Hadoop CombineFileInputFormat will
       // pass the path part only to accept().
       // Trailing the path with a separator to prevent partial matching.
@@ -559,7 +556,8 @@ public class CombineHiveInputFormat<K ex
     }
 
     public void addPath(Path p) {
-      filterPaths.add(p);
+      String pString = p.toUri().getPath().toString();
+      pStrings.add(pString);
     }
 
     // returns true if the specified path matches the prefix stored
@@ -567,7 +565,7 @@ public class CombineHiveInputFormat<K ex
     public boolean accept(Path path) {
       boolean find = false;
       while (path != null && !find) {
-        if(checkFilterPathContains(filterPaths, path)){
+        if(pStrings.contains(path.toString())) {
           find = true;
           break;
         }
@@ -576,31 +574,12 @@ public class CombineHiveInputFormat<K ex
       return find;
     }
 
-    private boolean checkFilterPathContains(Set<Path> filterPaths, Path path) {
-      for(Path filterPath : filterPaths){
-        try {
-          //if path does not have scheme, use the scheme from filterPath
-          //for comparison
-          path = path.makeQualified(filterPath.getFileSystem(conf));
-        } catch (IOException e) {
-          //this is not supposed to happen
-          //if it happens it can lead to incorrect splits being generated
-          // and files that should not be combined could end up being combied
-          throw new RuntimeException("Error making path " + path + " qualified ", e);
-        }
-        if(filterPath.equals(path)){
-          return true;
-        }
-      }
-      return false;
-    }
-
     @Override
     public String toString() {
       StringBuilder s = new StringBuilder();
       s.append("PathFilter: ");
-      for (Path filterPath : filterPaths) {
-        s.append(filterPath + " ");
+      for (String pString : pStrings) {
+        s.append(pString + " ");
       }
       return s.toString();
     }