svn commit: r1401935 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java

[ha...@apache.org]

Author: hashutosh
Date: Thu Oct 25 00:40:32 2012
New Revision: 1401935

URL: http://svn.apache.org/viewvc?rev=1401935&view=rev
Log:
HIVE-3519 : partition to directory comparison in CombineHiveInputFormat needs to accept partitions dir without scheme (Thejas Nair via Ashutosh Chauhan)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java?rev=1401935&r1=1401934&r2=1401935&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java Thu Oct 25 00:40:32 2012
@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.Set;
 
@@ -48,6 +49,7 @@ import org.apache.hadoop.mapred.OutputFo
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.util.Shell;
 
 /**
  * An util class for various Hive file format tasks.
@@ -265,8 +267,13 @@ public final class HiveFileFormatUtils {
     PartitionDesc part = doGetPartitionDescFromPath(pathToPartitionInfo, dir);
 
     if (part == null
-        && (ignoreSchema || (dir.toUri().getScheme() == null || dir.toUri().getScheme().trim()
-            .equals("")))) {
+        && (ignoreSchema
+            || (dir.toUri().getScheme() == null || dir.toUri().getScheme().trim()
+            .equals(""))
+            || pathsContainNoScheme(pathToPartitionInfo)
+            )
+
+        ) {
 
       Map<String, PartitionDesc> newPathToPartitionInfo = null;
       if (cacheMap != null) {
@@ -291,6 +298,17 @@ public final class HiveFileFormatUtils {
     }
   }
 
+  private static boolean pathsContainNoScheme(Map<String, PartitionDesc> pathToPartitionInfo) {
+
+    for( Entry<String, PartitionDesc> pe  : pathToPartitionInfo.entrySet()){
+      if(new Path(pe.getKey()).toUri().getScheme() != null){
+        return false;
+      }
+    }
+    return true;
+
+  }
+
   private static void populateNewPartitionDesc(
       Map<String, PartitionDesc> pathToPartitionInfo,
       Map<String, PartitionDesc> newPathToPartitionInfo) {
@@ -355,6 +373,11 @@ public final class HiveFileFormatUtils {
     }
 
     String dirPath = dir.toUri().getPath();
+    if(Shell.WINDOWS){
+      //temp hack
+      //do this to get rid of "/" before the drive letter in windows
+      dirPath = new Path(dirPath).toString();
+    }
     if (foundAlias(pathToAliases, dirPath)) {
       return dirPath;
     }

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java?rev=1401935&r1=1401934&r2=1401935&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java Thu Oct 25 00:40:32 2012
@@ -187,9 +187,10 @@ public class TestSymlinkTextInputFormat 
       assertEquals(1, retSplits.length);
     } catch (Exception e) {
       e.printStackTrace();
+      fail("Caught exception " + e); 
     } finally {
       if (tblCreated) {
-        drv.run("drop table text_symlink_text;").getResponseCode();
+        drv.run("drop table text_symlink_text").getResponseCode();
       }
     }
   }