[hive] 02/03: HIVE-23190: LLAP: modify IndexCache to pass filesystem object to TezSpillRecord (László Bodor reviewed by Rajesh Balamohan)

[ab...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git

commit 416edbb4a1a6148afe87a628589a6d2a6da84c91
Author: László Bodor <bo...@gmail.com>
AuthorDate: Wed Oct 28 22:04:22 2020 +0100

    HIVE-23190: LLAP: modify IndexCache to pass filesystem object to TezSpillRecord (László Bodor reviewed by Rajesh Balamohan)
    
    Signed-off-by: Laszlo Bodor <bo...@gmail.com>
---
 .../apache/hadoop/hive/llap/shufflehandler/IndexCache.java  | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java
index fb1bcfe..63c91da 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java
@@ -25,6 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.tez.runtime.library.common.Constants;
 import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;
@@ -42,12 +43,22 @@ class IndexCache {
 
   private final LinkedBlockingQueue<String> queue =
       new LinkedBlockingQueue<String>();
+  private FileSystem fs;
   public static final String INDEX_CACHE_MB = "llap.shuffle.indexcache.mb";
 
   public IndexCache(Configuration conf) {
     this.conf = conf;
     totalMemoryAllowed = conf.getInt(INDEX_CACHE_MB, 10) * 1024 * 1024;
     LOG.info("IndexCache created with max memory = " + totalMemoryAllowed);
+    initLocalFs();
+  }
+
+  private void initLocalFs() {
+    try {
+      this.fs = FileSystem.getLocal(conf).getRaw();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
   }
 
   /**
@@ -119,7 +130,7 @@ class IndexCache {
     LOG.debug("IndexCache MISS: MapId " + mapId + " not found") ;
     TezSpillRecord tmp = null;
     try {
-      tmp = new TezSpillRecord(indexFileName, conf, expectedIndexOwner);
+      tmp = new TezSpillRecord(indexFileName, fs, expectedIndexOwner);
     } catch (Throwable e) {
       tmp = new TezSpillRecord(0);
       cache.remove(mapId);