[hive] branch master updated: HIVE-23468 : LLAP: Optimise OrcEncodedDataReader to avoid FS init to NN (Rajesh Balamohan via Ashutosh Chauhan)

[ha...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 03165e0  HIVE-23468 : LLAP: Optimise OrcEncodedDataReader to avoid FS init to NN (Rajesh Balamohan via Ashutosh Chauhan)
03165e0 is described below

commit 03165e00eb52d77062a741b576e48971827dfea3
Author: Rajesh Balamohan <rb...@apache.org>
AuthorDate: Sat Jun 13 14:08:43 2020 -0700

    HIVE-23468 : LLAP: Optimise OrcEncodedDataReader to avoid FS init to NN (Rajesh Balamohan via Ashutosh Chauhan)
    
    Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
---
 .../apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java   | 2 +-
 ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java            | 7 +++----
 .../hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java   | 2 +-
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index c90ff52..35d066a 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -537,7 +537,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
     path = split.getPath();
     if (fileKey instanceof Long && HiveConf.getBoolVar(
         daemonConf, ConfVars.LLAP_IO_USE_FILEID_PATH)) {
-      path = HdfsUtils.getFileIdPath(fsSupplier.get(), path, (long)fileKey);
+      path = HdfsUtils.getFileIdPath(path, (long)fileKey);
     }
     LlapIoImpl.ORC_LOGGER.trace("Creating reader for {} ({})", path, split.getPath());
     long startTime = counters.startTimeCounter();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
index 4de5c8c..f590eb6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
@@ -102,10 +102,9 @@ public class HdfsUtils {
   //       as public utility method in HDFS to obtain the inode-based path.
   private static String HDFS_ID_PATH_PREFIX = "/.reserved/.inodes/";
 
-  public static Path getFileIdPath(
-      FileSystem fileSystem, Path path, long fileId) {
-    return ((fileSystem instanceof DistributedFileSystem))
-        ? new Path(HDFS_ID_PATH_PREFIX + fileId) : path;
+  public static Path getFileIdPath(Path path, long fileId) {
+    // BI/ETL split strategies set fileId correctly when HDFS is used.
+    return (fileId > 0) ? new Path(HDFS_ID_PATH_PREFIX + fileId) : path;
   }
 
   public static boolean isDefaultFs(DistributedFileSystem fs) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
index 3fd93ca..a3bbb7b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
@@ -221,7 +221,7 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase
       FileSystem fs = file.getFileSystem(configuration);
       if (cacheKey instanceof Long && HiveConf.getBoolVar(
           cacheConf, ConfVars.LLAP_IO_USE_FILEID_PATH)) {
-        file = HdfsUtils.getFileIdPath(fs, file, (long)cacheKey);
+        file = HdfsUtils.getFileIdPath(file, (long)cacheKey);
       }
     }