svn commit: r1625603 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql: exec/FetchOperator.java io/AcidUtils.java

[ga...@apache.org]

Author: gates
Date: Wed Sep 17 14:12:19 2014
New Revision: 1625603

URL: http://svn.apache.org/r1625603
Log:
HIVE-8103 Read ACID tables with FetchOperator returns no rows (Alan Gates, reviewed by Ashutosh Chauhan)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=1625603&r1=1625602&r2=1625603&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Wed Sep 17 14:12:19 2014
@@ -38,6 +38,7 @@ import org.apache.hadoop.hive.common.Obj
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
 import org.apache.hadoop.hive.ql.exec.FooterBuffer;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader;
 import org.apache.hadoop.hive.ql.io.HiveInputFormat;
 import org.apache.hadoop.hive.ql.io.HiveRecordReader;
@@ -748,7 +749,8 @@ public class FetchOperator implements Se
    */
   private FileStatus[] listStatusUnderPath(FileSystem fs, Path p) throws IOException {
     boolean recursive = HiveConf.getBoolVar(job, HiveConf.ConfVars.HADOOPMAPREDINPUTDIRRECURSIVE);
-    if (!recursive) {
+    // If this is in acid format always read it recursively regardless of what the jobconf says.
+    if (!recursive && !AcidUtils.isAcid(p, job)) {
       return fs.listStatus(p);
     }
     List<FileStatus> results = new ArrayList<FileStatus>();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java?rev=1625603&r1=1625602&r2=1625603&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java Wed Sep 17 14:12:19 2014
@@ -305,6 +305,28 @@ public class AcidUtils {
   }
 
   /**
+   * Is the given directory in ACID format?
+   * @param directory the partition directory to check
+   * @param conf the query configuration
+   * @return true, if it is an ACID directory
+   * @throws IOException
+   */
+  public static boolean isAcid(Path directory,
+                               Configuration conf) throws IOException {
+    FileSystem fs = directory.getFileSystem(conf);
+    for(FileStatus file: fs.listStatus(directory)) {
+      String filename = file.getPath().getName();
+      if (filename.startsWith(BASE_PREFIX) ||
+          filename.startsWith(DELTA_PREFIX)) {
+        if (file.isDirectory()) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  /**
    * Get the ACID state of the given directory. It finds the minimal set of
    * base and diff directories. Note that because major compactions don't
    * preserve the history, we can't use a base directory that includes a