You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/07/27 22:48:44 UTC

[3/4] hive git commit: HIVE-11259 : LLAP: clean up ORC dependencies part 1 (Sergey Shelukhin, reviewed by Prasanth Jayachandran)

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
index c16be38..d51acb5 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
@@ -19,11 +19,10 @@ package org.apache.hadoop.hive.llap.io.decode;
 
 import java.io.IOException;
 
-import org.apache.hadoop.hive.llap.Consumer;
+import org.apache.hadoop.hive.common.io.storage_api.EncodedColumnBatch;
+import org.apache.hadoop.hive.common.io.storage_api.EncodedColumnBatch.ColumnStreamData;
 import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters;
-import org.apache.hadoop.hive.llap.io.api.EncodedColumnBatch;
 import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
 import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
 import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
 import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics;
@@ -31,6 +30,8 @@ import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.io.orc.CompressionCodec;
 import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl.OrcEncodedColumnBatch;
+import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
+import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
 import org.apache.hadoop.hive.ql.io.orc.EncodedTreeReaderFactory;
 import org.apache.hadoop.hive.ql.io.orc.OrcProto;
 import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
@@ -71,14 +72,14 @@ public class OrcEncodedDataConsumer
   protected void decodeBatch(OrcEncodedColumnBatch batch,
       Consumer<ColumnVectorBatch> downstreamConsumer) {
     long startTime = counters.startTimeCounter();
-    int currentStripeIndex = batch.batchKey.stripeIx;
+    int currentStripeIndex = batch.getBatchKey().stripeIx;
 
     boolean sameStripe = currentStripeIndex == previousStripeIndex;
 
     try {
       OrcStripeMetadata stripeMetadata = stripes[currentStripeIndex];
       // Get non null row count from root column, to get max vector batches
-      int rgIdx = batch.batchKey.rgIx;
+      int rgIdx = batch.getBatchKey().rgIx;
       long nonNullRowCount = -1;
       if (rgIdx == OrcEncodedColumnBatch.ALL_RGS) {
         nonNullRowCount = stripeMetadata.getRowCount();
@@ -88,7 +89,7 @@ public class OrcEncodedDataConsumer
       }
       int maxBatchesRG = (int) ((nonNullRowCount / VectorizedRowBatch.DEFAULT_SIZE) + 1);
       int batchSize = VectorizedRowBatch.DEFAULT_SIZE;
-      int numCols = batch.columnIxs.length;
+      int numCols = batch.getColumnIxs().length;
       if (columnReaders == null || !sameStripe) {
         this.columnReaders = EncodedTreeReaderFactory.createEncodedTreeReader(numCols,
             fileMetadata.getTypes(), stripeMetadata.getEncodings(), batch, codec, skipCorrupt);
@@ -106,10 +107,10 @@ public class OrcEncodedDataConsumer
         }
 
         ColumnVectorBatch cvb = cvbPool.take();
-        assert cvb.cols.length == batch.columnIxs.length; // Must be constant per split.
+        assert cvb.cols.length == batch.getColumnIxs().length; // Must be constant per split.
         cvb.size = batchSize;
 
-        for (int idx = 0; idx < batch.columnIxs.length; idx++) {
+        for (int idx = 0; idx < batch.getColumnIxs().length; idx++) {
           cvb.cols[idx] = (ColumnVector)columnReaders[idx].nextVector(cvb.cols[idx], batchSize);
         }
 
@@ -130,8 +131,8 @@ public class OrcEncodedDataConsumer
       EncodedColumnBatch<OrcBatchKey> batch, int numCols,
       OrcStripeMetadata stripeMetadata) throws IOException {
     for (int i = 0; i < numCols; i++) {
-      int columnIndex = batch.columnIxs[i];
-      int rowGroupIndex = batch.batchKey.rgIx;
+      int columnIndex = batch.getColumnIxs()[i];
+      int rowGroupIndex = batch.getBatchKey().rgIx;
       OrcProto.RowIndex rowIndex = stripeMetadata.getRowIndexes()[columnIndex];
       OrcProto.RowIndexEntry rowIndexEntry = rowIndex.getEntry(rowGroupIndex);
       columnReaders[i].seek(new RecordReaderImpl.PositionProviderImpl(rowIndexEntry));
@@ -142,9 +143,9 @@ public class OrcEncodedDataConsumer
       EncodedColumnBatch<OrcBatchKey> batch, boolean sameStripe, int numCols,
       OrcStripeMetadata stripeMetadata) throws IOException {
     for (int i = 0; i < numCols; i++) {
-      int columnIndex = batch.columnIxs[i];
-      int rowGroupIndex = batch.batchKey.rgIx;
-      EncodedColumnBatch.StreamBuffer[] streamBuffers = batch.columnData[i];
+      int columnIndex = batch.getColumnIxs()[i];
+      int rowGroupIndex = batch.getBatchKey().rgIx;
+      ColumnStreamData[] streamBuffers = batch.getColumnData()[i];
       OrcProto.RowIndex rowIndex = stripeMetadata.getRowIndexes()[columnIndex];
       OrcProto.RowIndexEntry rowIndexEntry = rowIndex.getEntry(rowGroupIndex);
       columnReaders[i].setBuffers(streamBuffers, sameStripe);

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index 5cf0780..f88a943 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -1,6 +1,7 @@
 package org.apache.hadoop.hive.llap.io.encoded;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -12,25 +13,29 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.CallableWithNdc;
+import org.apache.hadoop.hive.common.DiskRange;
+import org.apache.hadoop.hive.common.DiskRangeList;
+import org.apache.hadoop.hive.common.io.storage_api.Allocator;
+import org.apache.hadoop.hive.common.io.storage_api.DataCache;
+import org.apache.hadoop.hive.common.io.storage_api.DataReader;
+import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
+import org.apache.hadoop.hive.common.io.storage_api.EncodedColumnBatch.ColumnStreamData;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.Consumer;
 import org.apache.hadoop.hive.llap.ConsumerFeedback;
 import org.apache.hadoop.hive.llap.DebugUtils;
 import org.apache.hadoop.hive.llap.cache.Cache;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
 import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters;
 import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters.Counter;
-import org.apache.hadoop.hive.llap.io.api.EncodedColumnBatch.StreamBuffer;
-import org.apache.hadoop.hive.llap.io.api.cache.LlapMemoryBuffer;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache;
 import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcCacheKey;
 import org.apache.hadoop.hive.llap.io.decode.OrcEncodedDataConsumer;
 import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
 import org.apache.hadoop.hive.llap.io.metadata.OrcMetadataCache;
 import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
 import org.apache.hadoop.hive.ql.exec.DDLTask;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.io.HdfsUtils;
 import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
 import org.apache.hadoop.hive.ql.io.orc.EncodedReader;
 import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl;
@@ -44,9 +49,14 @@ import org.apache.hadoop.hive.ql.io.orc.OrcSplit;
 import org.apache.hadoop.hive.ql.io.orc.Reader;
 import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
 import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.SargApplier;
+import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
+import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
+import org.apache.hadoop.hive.ql.io.orc.llap.OrcCacheKey;
 import org.apache.hadoop.hive.ql.io.orc.RecordReaderUtils;
 import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.mapred.FileSplit;
 import org.apache.hadoop.mapred.InputSplit;
 
@@ -235,7 +245,9 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
     try {
       ensureOrcReader();
       // Reader creating updates HDFS counters, don't do it here.
-      stripeReader = orcReader.encodedReader(fileId, lowLevelCache, counters, dataConsumer);
+      DataWrapperForOrc dw = new DataWrapperForOrc();
+      stripeReader = orcReader.encodedReader(fileId, dw, dw);
+      stripeReader.setDebugTracing(DebugUtils.isTraceOrcEnabled());
     } catch (Throwable t) {
       consumer.setError(t);
       recordReaderTime(startTime);
@@ -340,7 +352,8 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
         // Also, currently readEncodedColumns is not stoppable. The consumer will discard the
         // data it receives for one stripe. We could probably interrupt it, if it checked that.
         stripeReader.readEncodedColumns(stripeIx, stripe, stripeMetadata.getRowIndexes(),
-            stripeMetadata.getEncodings(), stripeMetadata.getStreams(), stripeIncludes, colRgs);
+            stripeMetadata.getEncodings(), stripeMetadata.getStreams(), stripeIncludes,
+            colRgs, dataConsumer);
       } catch (Throwable t) {
         consumer.setError(t);
         cleanupReaders();
@@ -434,7 +447,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
       }
     }
     LOG.warn("Split for " + split.getPath() + " (" + split.getClass() + ") does not have file ID");
-    return RecordReaderUtils.getFileId(fs, split.getPath());
+    return HdfsUtils.getFileId(fs, split.getPath());
   }
 
   private boolean[][] genStripeColRgs(List<Integer> stripeCols, boolean[][] globalColRgs) {
@@ -499,7 +512,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
    */
   private void ensureOrcReader() throws IOException {
     if (orcReader != null) return;
-    Path path = RecordReaderUtils.getFileIdPath(fs, split.getPath(), fileId);
+    Path path = HdfsUtils.getFileIdPath(fs, split.getPath(), fileId);
     if (DebugUtils.isTraceOrcEnabled()) {
       LOG.info("Creating reader for " + path + " (" + split.getPath() + ")");
     }
@@ -578,15 +591,15 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
 
   @Override
   public void returnData(OrcEncodedColumnBatch ecb) {
-    for (StreamBuffer[] datas : ecb.columnData) {
-      for (StreamBuffer data : datas) {
+    for (ColumnStreamData[] datas : ecb.getColumnData()) {
+      for (ColumnStreamData data : datas) {
         if (data.decRef() != 0) continue;
         if (DebugUtils.isTraceLockingEnabled()) {
-          for (LlapMemoryBuffer buf : data.cacheBuffers) {
+          for (MemoryBuffer buf : data.getCacheBuffers()) {
             LlapIoImpl.LOG.info("Unlocking " + buf + " at the end of processing");
           }
         }
-        lowLevelCache.releaseBuffers(data.cacheBuffers);
+        lowLevelCache.releaseBuffers(data.getCacheBuffers());
         EncodedReaderImpl.SB_POOL.offer(data);
       }
     }
@@ -743,12 +756,12 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
             if ((readMask == SargApplier.READ_NO_RGS) || (readMask != SargApplier.READ_ALL_RGS
                 && (readMask.length <= rgIx || !readMask[rgIx]))) continue;
             key.colIx = columnIds.get(colIxMod);
-            StreamBuffer[] cached = cache.get(key);
+            ColumnStreamData[] cached = cache.get(key);
             if (cached == null) {
               isMissingAnyRgs[colIxMod] = true;
               continue;
             }
-            col.setAllStreams(colIxMod, key.colIx, cached);
+            col.setAllStreamsData(colIxMod, key.colIx, cached);
             hasAnyCached = true;
             if (readMask == SargApplier.READ_ALL_RGS) {
               // We were going to read all RGs, but some were in cache, allocate the mask.
@@ -793,16 +806,16 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
   public void consumeData(OrcEncodedColumnBatch data) {
     // Store object in cache; create new key object - cannot be reused.
     assert cache != null;
-    for (int i = 0; i < data.columnData.length; ++i) {
-      OrcCacheKey key = new OrcCacheKey(data.batchKey, data.columnIxs[i]);
-      StreamBuffer[] toCache = data.columnData[i];
-      StreamBuffer[] cached = cache.cacheOrGet(key, toCache);
+    for (int i = 0; i < data.getColumnData().length; ++i) {
+      OrcCacheKey key = new OrcCacheKey(data.getBatchKey(), data.getColumnIxs()[i]);
+      ColumnStreamData[] toCache = data.getColumnData()[i];
+      ColumnStreamData[] cached = cache.cacheOrGet(key, toCache);
       if (toCache != cached) {
-        for (StreamBuffer sb : toCache) {
+        for (ColumnStreamData sb : toCache) {
           if (sb.decRef() != 0) continue;
-          lowLevelCache.releaseBuffers(sb.cacheBuffers);
+          lowLevelCache.releaseBuffers(sb.getCacheBuffers());
         }
-        data.columnData[i] = cached;
+        data.getColumnData()[i] = cached;
       }
     }
     consumer.consumeData(data);
@@ -812,4 +825,82 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
   public void setError(Throwable t) {
     consumer.setError(t);
   }
+
+  private class DataWrapperForOrc implements DataReader, DataCache {
+    private DataReader orcDataReader;
+
+    public DataWrapperForOrc() {
+      boolean useZeroCopy = (conf != null)
+          && HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ORC_ZEROCOPY);
+      if (useZeroCopy && !lowLevelCache.getAllocator().isDirectAlloc()) {
+        throw new UnsupportedOperationException("Cannot use zero-copy reader with non-direct cache "
+            + "buffers; either disable zero-copy or enable direct cache allocation");
+      }
+      this.orcDataReader = orcReader.createDefaultDataReader(useZeroCopy);
+    }
+
+    @Override
+    public DiskRangeList getFileData(long fileId, DiskRangeList range,
+        long baseOffset, DiskRangeListFactory factory, BooleanRef gotAllData) {
+      return lowLevelCache.getFileData(fileId, range, baseOffset, factory, counters, gotAllData);
+    }
+
+    @Override
+    public long[] putFileData(long fileId, DiskRange[] ranges,
+        MemoryBuffer[] data, long baseOffset) {
+      return lowLevelCache.putFileData(
+          fileId, ranges, data, baseOffset, Priority.NORMAL, counters);
+    }
+
+    @Override
+    public void releaseBuffer(MemoryBuffer buffer) {
+      lowLevelCache.releaseBuffer(buffer);
+    }
+
+    @Override
+    public void reuseBuffer(MemoryBuffer buffer) {
+      boolean isReused = lowLevelCache.reuseBuffer(buffer);
+      assert isReused;
+    }
+
+    @Override
+    public Allocator getAllocator() {
+      return lowLevelCache.getAllocator();
+    }
+
+    @Override
+    public void close() throws IOException {
+      orcDataReader.close();
+    }
+
+    @Override
+    public DiskRangeList readFileData(DiskRangeList range, long baseOffset,
+        boolean doForceDirect) throws IOException {
+      long startTime = counters.startTimeCounter();
+      DiskRangeList result = orcDataReader.readFileData(range, baseOffset, doForceDirect);
+      counters.recordHdfsTime(startTime);
+      if (DebugUtils.isTraceOrcEnabled() && LOG.isInfoEnabled()) {
+        LOG.info("Disk ranges after disk read (file " + fileId + ", base offset " + baseOffset
+              + "): " + RecordReaderUtils.stringifyDiskRanges(result));
+      }
+      return result;
+    }
+
+    @Override
+    public boolean isTrackingDiskRanges() {
+      return orcDataReader.isTrackingDiskRanges();
+    }
+
+    @Override
+    public void releaseBuffer(ByteBuffer buffer) {
+      orcDataReader.releaseBuffer(buffer);
+    }
+
+    @Override
+    public void open() throws IOException {
+      long startTime = counters.startTimeCounter();
+      orcDataReader.open();
+      counters.recordHdfsTime(startTime);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
index dbfe4df..635ba6c 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
@@ -21,13 +21,10 @@ package org.apache.hadoop.hive.llap.io.metadata;
 import java.io.IOException;
 import java.util.concurrent.ConcurrentHashMap;
 
-import org.apache.hadoop.hive.llap.cache.EvictionListener;
-import org.apache.hadoop.hive.llap.cache.LlapCacheableBuffer;
-import org.apache.hadoop.hive.llap.cache.LowLevelCacheMemoryManager;
 import org.apache.hadoop.hive.llap.cache.LowLevelCachePolicy;
 import org.apache.hadoop.hive.llap.cache.MemoryManager;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache.Priority;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
+import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
 
 public class OrcMetadataCache {
   private final ConcurrentHashMap<Long, OrcFileMetadata> metadata =

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
index c0b8b34..762c9f9 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator;
 import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator.ObjectEstimator;
 import org.apache.hadoop.hive.llap.cache.EvictionDispatcher;
 import org.apache.hadoop.hive.llap.cache.LlapCacheableBuffer;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
 import org.apache.hadoop.hive.ql.io.orc.MetadataReader;
 import org.apache.hadoop.hive.ql.io.orc.OrcProto.BloomFilter;
 import org.apache.hadoop.hive.ql.io.orc.OrcProto.BloomFilterIndex;
@@ -37,6 +36,7 @@ import org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream;
 import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter;
 import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
 import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
+import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
 
 import com.google.common.annotations.VisibleForTesting;
 

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
index 50d5e19..3299135 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
@@ -30,9 +30,9 @@ import java.util.concurrent.FutureTask;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
+import org.apache.hadoop.hive.common.io.storage_api.Allocator.AllocatorOutOfMemoryException;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.cache.Allocator.LlapCacheOutOfMemoryException;
-import org.apache.hadoop.hive.llap.io.api.cache.LlapMemoryBuffer;
 import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
 import org.junit.Test;
 
@@ -191,7 +191,7 @@ public class TestBuddyAllocator {
   }
 
   private void allocSameSize(BuddyAllocator a, int allocCount, int sizeLog2) throws Exception {
-    LlapMemoryBuffer[][] allocs = new LlapMemoryBuffer[allocCount][];
+    MemoryBuffer[][] allocs = new MemoryBuffer[allocCount][];
     long[][] testValues = new long[allocCount][];
     for (int j = 0; j < allocCount; ++j) {
       allocateAndUseBuffer(a, allocs, testValues, 1, j, sizeLog2);
@@ -202,7 +202,7 @@ public class TestBuddyAllocator {
   private void allocateUp(BuddyAllocator a, int min, int max, int allocPerSize,
       boolean isSameOrderDealloc) throws Exception {
     int sizes = max - min + 1;
-    LlapMemoryBuffer[][] allocs = new LlapMemoryBuffer[sizes][];
+    MemoryBuffer[][] allocs = new MemoryBuffer[sizes][];
     // Put in the beginning; relies on the knowledge of internal implementation. Pave?
     long[][] testValues = new long[sizes][];
     for (int i = min; i <= max; ++i) {
@@ -214,7 +214,7 @@ public class TestBuddyAllocator {
   private void allocateDown(BuddyAllocator a, int min, int max, int allocPerSize,
       boolean isSameOrderDealloc) throws Exception {
     int sizes = max - min + 1;
-    LlapMemoryBuffer[][] allocs = new LlapMemoryBuffer[sizes][];
+    MemoryBuffer[][] allocs = new MemoryBuffer[sizes][];
     // Put in the beginning; relies on the knowledge of internal implementation. Pave?
     long[][] testValues = new long[sizes][];
     for (int i = max; i >= min; --i) {
@@ -223,20 +223,20 @@ public class TestBuddyAllocator {
     deallocUpOrDown(a, isSameOrderDealloc, allocs, testValues);
   }
 
-  private void allocateAndUseBuffer(BuddyAllocator a, LlapMemoryBuffer[][] allocs,
+  private void allocateAndUseBuffer(BuddyAllocator a, MemoryBuffer[][] allocs,
       long[][] testValues, int allocCount, int index, int sizeLog2) throws Exception {
-    allocs[index] = new LlapMemoryBuffer[allocCount];
+    allocs[index] = new MemoryBuffer[allocCount];
     testValues[index] = new long[allocCount];
     int size = (1 << sizeLog2) - 1;
     try {
       a.allocateMultiple(allocs[index], size);
-    } catch (LlapCacheOutOfMemoryException ex) {
+    } catch (AllocatorOutOfMemoryException ex) {
       LOG.error("Failed to allocate " + allocCount + " of " + size + "; " + a.debugDump());
       throw ex;
     }
     // LOG.info("Allocated " + allocCount + " of " + size + "; " + a.debugDump());
     for (int j = 0; j < allocCount; ++j) {
-      LlapMemoryBuffer mem = allocs[index][j];
+      MemoryBuffer mem = allocs[index][j];
       long testValue = testValues[index][j] = rdm.nextLong();
       int pos = mem.getByteBufferRaw().position();
       mem.getByteBufferRaw().putLong(pos, testValue);
@@ -248,7 +248,7 @@ public class TestBuddyAllocator {
   }
 
   private void deallocUpOrDown(BuddyAllocator a, boolean isSameOrderDealloc,
-      LlapMemoryBuffer[][] allocs, long[][] testValues) {
+      MemoryBuffer[][] allocs, long[][] testValues) {
     if (isSameOrderDealloc) {
       for (int i = 0; i < allocs.length; ++i) {
         deallocBuffers(a, allocs[i], testValues[i]);
@@ -261,7 +261,7 @@ public class TestBuddyAllocator {
   }
 
   private void deallocBuffers(
-      BuddyAllocator a, LlapMemoryBuffer[] allocs, long[] testValues) {
+      BuddyAllocator a, MemoryBuffer[] allocs, long[] testValues) {
     for (int j = 0; j < allocs.length; ++j) {
       LlapDataBuffer mem = (LlapDataBuffer)allocs[j];
       int pos = mem.getByteBufferRaw().position();

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
index 4697445..382e948 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
@@ -30,7 +30,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator;
 import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator.ObjectEstimator;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
 import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
 import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
 import org.apache.hadoop.hive.ql.io.orc.MetadataReader;
@@ -54,6 +53,7 @@ import org.apache.hadoop.hive.ql.io.orc.OrcProto.TimestampStatistics;
 import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
 import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.Index;
 import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
+import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
 import org.apache.hadoop.hive.ql.util.JavaDataModel;
 import org.junit.Test;
 import org.mockito.Mockito;

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
index 2c87ec1..d4b8a46 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
@@ -35,26 +35,28 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.common.DiskRange;
 import org.apache.hadoop.hive.common.DiskRangeList;
-import org.apache.hadoop.hive.common.DiskRangeList.DiskRangeListCreateHelper;
-import org.apache.hadoop.hive.llap.io.api.cache.LlapMemoryBuffer;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache.CacheChunkFactory;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache.Priority;
+import org.apache.hadoop.hive.common.DiskRangeList.CreateHelper;
+import org.apache.hadoop.hive.common.io.storage_api.DataCache.DiskRangeListFactory;
+import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
 import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.CacheChunk;
+import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl.CacheChunk;
 import org.junit.Test;
 
 public class TestLowLevelCacheImpl {
   private static final Log LOG = LogFactory.getLog(TestLowLevelCacheImpl.class);
 
-  private static final CacheChunkFactory testFactory = new CacheChunkFactory() {
-    public DiskRangeList createCacheChunk(LlapMemoryBuffer buffer, long offset, long end) {
-      return new CacheChunk(buffer, offset, end);
+  private static final DiskRangeListFactory testFactory = new DiskRangeListFactory() {
+    public DiskRangeList createCacheChunk(MemoryBuffer buffer, long offset, long end) {
+      CacheChunk cc = new CacheChunk();
+      cc.init(buffer, offset, end);
+      return cc;
     }
   };
 
   private static class DummyAllocator implements EvictionAwareAllocator {
     @Override
-    public void allocateMultiple(LlapMemoryBuffer[] dest, int size) {
+    public void allocateMultiple(MemoryBuffer[] dest, int size) {
       for (int i = 0; i < dest.length; ++i) {
         LlapDataBuffer buf = new LlapDataBuffer();
         buf.initialize(0, null, -1, size);
@@ -63,11 +65,11 @@ public class TestLowLevelCacheImpl {
     }
 
     @Override
-    public void deallocate(LlapMemoryBuffer buffer) {
+    public void deallocate(MemoryBuffer buffer) {
     }
 
     @Override
-    public void deallocateEvicted(LlapMemoryBuffer buffer) {
+    public void deallocateEvicted(MemoryBuffer buffer) {
     }
 
     @Override
@@ -79,6 +81,11 @@ public class TestLowLevelCacheImpl {
     public int getMaxAllocation() {
       return 0;
     }
+
+    @Override
+    public MemoryBuffer createUnallocated() {
+      return new LlapDataBuffer();
+    }
   }
 
   private static class DummyCachePolicy implements LowLevelCachePolicy {
@@ -115,16 +122,16 @@ public class TestLowLevelCacheImpl {
         LlapDaemonCacheMetrics.create("test", "1"), new DummyCachePolicy(),
         new DummyAllocator(), true, -1); // no cleanup thread
     long fn1 = 1, fn2 = 2;
-    LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[] { fb(), fb(), fb(), fb(), fb(), fb() };
+    MemoryBuffer[] fakes = new MemoryBuffer[] { fb(), fb(), fb(), fb(), fb(), fb() };
     verifyRefcount(fakes, 1, 1, 1, 1, 1, 1);
-    assertNull(cache.putFileData(fn1, drs(1, 2), fbs(fakes, 0, 1), 0, Priority.NORMAL));
-    assertNull(cache.putFileData(fn2, drs(1, 2), fbs(fakes, 2, 3), 0, Priority.NORMAL));
+    assertNull(cache.putFileData(fn1, drs(1, 2), fbs(fakes, 0, 1), 0, Priority.NORMAL, null));
+    assertNull(cache.putFileData(fn2, drs(1, 2), fbs(fakes, 2, 3), 0, Priority.NORMAL, null));
     verifyCacheGet(cache, fn1, 1, 3, fakes[0], fakes[1]);
     verifyCacheGet(cache, fn2, 1, 3, fakes[2], fakes[3]);
     verifyCacheGet(cache, fn1, 2, 4, fakes[1], dr(3, 4));
     verifyRefcount(fakes, 3, 4, 3, 3, 1, 1);
-    LlapMemoryBuffer[] bufsDiff = fbs(fakes, 4, 5);
-    long[] mask = cache.putFileData(fn1, drs(3, 1), bufsDiff, 0, Priority.NORMAL);
+    MemoryBuffer[] bufsDiff = fbs(fakes, 4, 5);
+    long[] mask = cache.putFileData(fn1, drs(3, 1), bufsDiff, 0, Priority.NORMAL, null);
     assertEquals(1, mask.length);
     assertEquals(2, mask[0]); // 2nd bit set - element 2 was already in cache.
     assertSame(fakes[0], bufsDiff[1]); // Should have been replaced
@@ -134,7 +141,7 @@ public class TestLowLevelCacheImpl {
   }
 
   private void verifyCacheGet(LowLevelCacheImpl cache, long fileId, Object... stuff) {
-    DiskRangeListCreateHelper list = new DiskRangeListCreateHelper();
+    CreateHelper list = new CreateHelper();
     DiskRangeList iter = null;
     int intCount = 0, lastInt = -1;
     int resultCount = stuff.length;
@@ -153,13 +160,13 @@ public class TestLowLevelCacheImpl {
       } else if (intCount >= 0) {
         assertTrue(intCount == 0);
         intCount = -1;
-        iter = cache.getFileData(fileId, list.get(), 0, testFactory);
+        iter = cache.getFileData(fileId, list.get(), 0, testFactory, null, null);
         assertEquals(resultCount, iter.listSize());
       }
       assertTrue(iter != null);
-      if (obj instanceof LlapMemoryBuffer) {
+      if (obj instanceof MemoryBuffer) {
         assertTrue(iter instanceof CacheChunk);
-        assertSame(obj, ((CacheChunk)iter).buffer);
+        assertSame(obj, ((CacheChunk)iter).getBuffer());
       } else {
         assertTrue(iter.equals(obj));
       }
@@ -173,9 +180,9 @@ public class TestLowLevelCacheImpl {
         LlapDaemonCacheMetrics.create("test", "1"), new DummyCachePolicy(),
         new DummyAllocator(), true, -1); // no cleanup thread
     long fn = 1;
-    LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[] { fb(), fb() };
+    MemoryBuffer[] fakes = new MemoryBuffer[] { fb(), fb() };
     assertNull(cache.putFileData(
-        fn, new DiskRange[] { dr(2, 4), dr(6, 8) }, fakes, 0, Priority.NORMAL));
+        fn, new DiskRange[] { dr(2, 4), dr(6, 8) }, fakes, 0, Priority.NORMAL, null));
     verifyCacheGet(cache, fn, 1, 9, dr(1, 2), fakes[0], dr(4, 6), fakes[1], dr(8, 9));
     verifyCacheGet(cache, fn, 2, 8, fakes[0], dr(4, 6), fakes[1]);
     verifyCacheGet(cache, fn, 1, 5, dr(1, 2), fakes[0], dr(4, 5));
@@ -192,9 +199,9 @@ public class TestLowLevelCacheImpl {
         LlapDaemonCacheMetrics.create("test", "1"), new DummyCachePolicy(),
         new DummyAllocator(), false, -1); // no cleanup thread
     long fn = 1;
-    LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[] { fb(), fb() };
+    MemoryBuffer[] fakes = new MemoryBuffer[] { fb(), fb() };
     assertNull(cache.putFileData(
-        fn, new DiskRange[] { dr(2, 4), dr(6, 8) }, fakes, 0, Priority.NORMAL));
+        fn, new DiskRange[] { dr(2, 4), dr(6, 8) }, fakes, 0, Priority.NORMAL, null));
     // We expect cache requests from the middle here
     verifyCacheGet(cache, fn, 3, 4, fakes[0]);
     verifyCacheGet(cache, fn, 3, 7, fakes[0], dr(4, 6), fakes[1]);
@@ -206,9 +213,9 @@ public class TestLowLevelCacheImpl {
         LlapDaemonCacheMetrics.create("test", "1"), new DummyCachePolicy(),
         new DummyAllocator(), true, -1); // no cleanup thread
     long fn1 = 1, fn2 = 2;
-    LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[] { fb(), fb(), fb() };
-    assertNull(cache.putFileData(fn1, drs(1, 2), fbs(fakes, 0, 1), 0, Priority.NORMAL));
-    assertNull(cache.putFileData(fn2, drs(1), fbs(fakes, 2), 0, Priority.NORMAL));
+    MemoryBuffer[] fakes = new MemoryBuffer[] { fb(), fb(), fb() };
+    assertNull(cache.putFileData(fn1, drs(1, 2), fbs(fakes, 0, 1), 0, Priority.NORMAL, null));
+    assertNull(cache.putFileData(fn2, drs(1), fbs(fakes, 2), 0, Priority.NORMAL, null));
     verifyCacheGet(cache, fn1, 1, 3, fakes[0], fakes[1]);
     verifyCacheGet(cache, fn2, 1, 2, fakes[2]);
     verifyRefcount(fakes, 3, 3, 3);
@@ -225,23 +232,23 @@ public class TestLowLevelCacheImpl {
         LlapDaemonCacheMetrics.create("test", "1"), new DummyCachePolicy(),
         new DummyAllocator(), true, -1); // no cleanup thread
     long fn1 = 1, fn2 = 2;
-    LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[] {
+    MemoryBuffer[] fakes = new MemoryBuffer[] {
         fb(), fb(), fb(), fb(), fb(), fb(), fb(), fb(), fb() };
-    assertNull(cache.putFileData(fn1, drs(1, 2, 3), fbs(fakes, 0, 1, 2), 0, Priority.NORMAL));
-    assertNull(cache.putFileData(fn2, drs(1), fbs(fakes, 3), 0, Priority.NORMAL));
+    assertNull(cache.putFileData(fn1, drs(1, 2, 3), fbs(fakes, 0, 1, 2), 0, Priority.NORMAL, null));
+    assertNull(cache.putFileData(fn2, drs(1), fbs(fakes, 3), 0, Priority.NORMAL, null));
     evict(cache, fakes[0]);
     evict(cache, fakes[3]);
     long[] mask = cache.putFileData(
-        fn1, drs(1, 2, 3, 4), fbs(fakes, 4, 5, 6, 7), 0, Priority.NORMAL);
+        fn1, drs(1, 2, 3, 4), fbs(fakes, 4, 5, 6, 7), 0, Priority.NORMAL, null);
     assertEquals(1, mask.length);
     assertEquals(6, mask[0]); // Buffers at offset 2 & 3 exist; 1 exists and is stale; 4 doesn't
-    assertNull(cache.putFileData(fn2, drs(1), fbs(fakes, 8), 0, Priority.NORMAL));
+    assertNull(cache.putFileData(fn2, drs(1), fbs(fakes, 8), 0, Priority.NORMAL, null));
     verifyCacheGet(cache, fn1, 1, 5, fakes[4], fakes[1], fakes[2], fakes[7]);
   }
 
   @Test
   public void testCacheMetrics() {
-    DiskRangeListCreateHelper list = new DiskRangeListCreateHelper();
+    CreateHelper list = new CreateHelper();
     list.addOrMerge(0, 100, true, false);
     list.addOrMerge(100, 200, true, false);
     list.addOrMerge(200, 300, true, false);
@@ -249,7 +256,7 @@ public class TestLowLevelCacheImpl {
     list.addOrMerge(400, 500, true, false);
     assertEquals(1, list.get().listSize());
     assertEquals(500, list.get().getTotalLength());
-    list = new DiskRangeListCreateHelper();
+    list = new CreateHelper();
     list.addOrMerge(0, 100, false, false);
     list.addOrMerge(100, 200, false, false);
     list.addOrMerge(200, 300, false, false);
@@ -257,7 +264,7 @@ public class TestLowLevelCacheImpl {
     list.addOrMerge(400, 500, false, false);
     assertEquals(5, list.get().listSize());
     assertEquals(500, list.get().getTotalLength());
-    list = new DiskRangeListCreateHelper();
+    list = new CreateHelper();
     list.addOrMerge(0, 100, true, false);
     list.addOrMerge(100, 200, true, false);
     list.addOrMerge(200, 300, false, false);
@@ -270,35 +277,35 @@ public class TestLowLevelCacheImpl {
     LowLevelCacheImpl cache = new LowLevelCacheImpl(metrics,
         new DummyCachePolicy(), new DummyAllocator(), true, -1); // no cleanup thread
     long fn = 1;
-    LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[]{fb(), fb(), fb()};
+    MemoryBuffer[] fakes = new MemoryBuffer[]{fb(), fb(), fb()};
     cache.putFileData(fn, new DiskRange[]{dr(0, 100), dr(300, 500), dr(800, 1000)},
-        fakes, 0, Priority.NORMAL);
+        fakes, 0, Priority.NORMAL, null);
     assertEquals(0, metrics.getCacheRequestedBytes());
     assertEquals(0, metrics.getCacheHitBytes());
-    list = new DiskRangeListCreateHelper();
+    list = new CreateHelper();
     list.addOrMerge(0, 1000, true, false);
-    cache.getFileData(fn, list.get(), 0, testFactory);
+    cache.getFileData(fn, list.get(), 0, testFactory, null, null);
     assertEquals(1000, metrics.getCacheRequestedBytes());
     assertEquals(500, metrics.getCacheHitBytes());
 
-    list = new DiskRangeListCreateHelper();
+    list = new CreateHelper();
     list.addOrMerge(0, 100, true, false);
-    cache.getFileData(fn, list.get(), 0, testFactory);
+    cache.getFileData(fn, list.get(), 0, testFactory, null, null);
     assertEquals(1100, metrics.getCacheRequestedBytes());
     assertEquals(600, metrics.getCacheHitBytes());
 
-    list = new DiskRangeListCreateHelper();
+    list = new CreateHelper();
     list.addOrMerge(0, 100, true, false);
     list.addOrMerge(300, 500, true, false);
     list.addOrMerge(800, 1000, true, false);
-    cache.getFileData(fn, list.get(), 0, testFactory);
+    cache.getFileData(fn, list.get(), 0, testFactory, null, null);
     assertEquals(1600, metrics.getCacheRequestedBytes());
     assertEquals(1100, metrics.getCacheHitBytes());
 
-    list = new DiskRangeListCreateHelper();
+    list = new CreateHelper();
     list.addOrMerge(300, 500, true, false);
     list.addOrMerge(1000, 2000, true, false);
-    cache.getFileData(fn, list.get(), 0, testFactory);
+    cache.getFileData(fn, list.get(), 0, testFactory, null, null);
     assertEquals(2800, metrics.getCacheRequestedBytes());
     assertEquals(1300, metrics.getCacheHitBytes());
   }
@@ -325,12 +332,12 @@ public class TestLowLevelCacheImpl {
             if (isGet) {
               int[] offsets = new int[count];
               count = generateOffsets(offsetsToUse, rdm, offsets);
-              DiskRangeListCreateHelper list = new DiskRangeListCreateHelper();
+              CreateHelper list = new CreateHelper();
               for (int j = 0; i < count; ++i) {
                 list.addOrMerge(offsets[j], offsets[j] + 1, true, false);
               }
 
-              DiskRangeList iter = cache.getFileData(fileName, list.get(), 0, testFactory);
+              DiskRangeList iter = cache.getFileData(fileName, list.get(), 0, testFactory, null, null);
               int j = -1;
               while (iter != null) {
                 ++j;
@@ -339,7 +346,7 @@ public class TestLowLevelCacheImpl {
                   continue;
                 }
                 ++gets;
-                LlapDataBuffer result = (LlapDataBuffer)((CacheChunk)iter).buffer;
+                LlapDataBuffer result = (LlapDataBuffer)((CacheChunk)iter).getBuffer();
                 assertEquals(makeFakeArenaIndex(fileIndex, offsets[j]), result.arenaIndex);
                 cache.releaseBuffer(result);
                 iter = iter.next;
@@ -352,13 +359,13 @@ public class TestLowLevelCacheImpl {
                 ranges[j] = dr(next, next + 1);
                 offsets[j] = next;
               }
-              LlapMemoryBuffer[] buffers = new LlapMemoryBuffer[count];
+              MemoryBuffer[] buffers = new MemoryBuffer[count];
               for (int j = 0; j < offsets.length; ++j) {
                 LlapDataBuffer buf = LowLevelCacheImpl.allocateFake();
                 buf.arenaIndex = makeFakeArenaIndex(fileIndex, offsets[j]);
                 buffers[j] = buf;
               }
-              long[] mask = cache.putFileData(fileName, ranges, buffers, 0, Priority.NORMAL);
+              long[] mask = cache.putFileData(fileName, ranges, buffers, 0, Priority.NORMAL, null);
               puts += buffers.length;
               long maskVal = 0;
               if (mask != null) {
@@ -396,14 +403,14 @@ public class TestLowLevelCacheImpl {
           DiskRangeList head = new DiskRangeList(0, offsetsToUse + 1);
           isFirstFile = !isFirstFile;
           long fileId = isFirstFile ? fn1 : fn2;
-          head = cache.getFileData(fileId, head, 0, testFactory);
+          head = cache.getFileData(fileId, head, 0, testFactory, null, null);
           DiskRange[] results = head.listToArray();
           int startIndex = rdm.nextInt(results.length), index = startIndex;
           LlapDataBuffer victim = null;
           do {
             DiskRange r = results[index];
             if (r instanceof CacheChunk) {
-              LlapDataBuffer result = (LlapDataBuffer)((CacheChunk)r).buffer;
+              LlapDataBuffer result = (LlapDataBuffer)((CacheChunk)r).getBuffer();
               cache.releaseBuffer(result);
               if (victim == null && result.invalidate()) {
                 ++evictions;
@@ -452,7 +459,7 @@ public class TestLowLevelCacheImpl {
     }
   }
 
-  private void evict(LowLevelCacheImpl cache, LlapMemoryBuffer fake) {
+  private void evict(LowLevelCacheImpl cache, MemoryBuffer fake) {
     LlapDataBuffer victimBuffer = (LlapDataBuffer)fake;
     int refCount = victimBuffer.getRefCount();
     for (int i = 0; i < refCount; ++i) {
@@ -462,14 +469,14 @@ public class TestLowLevelCacheImpl {
     cache.notifyEvicted(victimBuffer);
   }
 
-  private void verifyRefcount(LlapMemoryBuffer[] fakes, int... refCounts) {
+  private void verifyRefcount(MemoryBuffer[] fakes, int... refCounts) {
     for (int i = 0; i < refCounts.length; ++i) {
       assertEquals("At " + i, refCounts[i], ((LlapDataBuffer)fakes[i]).getRefCount());
     }
   }
 
-  private LlapMemoryBuffer[] fbs(LlapMemoryBuffer[] fakes, int... indexes) {
-    LlapMemoryBuffer[] rv = new LlapMemoryBuffer[indexes.length];
+  private MemoryBuffer[] fbs(MemoryBuffer[] fakes, int... indexes) {
+    MemoryBuffer[] rv = new MemoryBuffer[indexes.length];
     for (int i = 0; i < indexes.length; ++i) {
       rv[i] = (indexes[i] == -1) ? null : fakes[indexes[i]];
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
index 7349208..a423eeb 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
@@ -33,7 +33,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache.Priority;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
 import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
 import org.junit.Assume;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
index 3549fd9..2886d54 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.*;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache.Priority;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
 import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
 import org.apache.hadoop.hive.llap.io.metadata.OrcMetadataCache;
 import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java b/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
index 8b61fe4..8dc62d8 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
@@ -35,10 +35,6 @@ public class DebugUtils {
     return isTraceOrcEnabled; // TODO: temporary, should be hardcoded false
   }
 
-  public static boolean isTraceRangesEnabled() {
-    return false;
-  }
-
   public static boolean isTraceLockingEnabled() {
     return false;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
new file mode 100644
index 0000000..0095d31
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
+
+public class HdfsUtils {
+  private static final HadoopShims SHIMS = ShimLoader.getHadoopShims();
+  private static final Log LOG = LogFactory.getLog(HdfsUtils.class);
+
+  public static long getFileId(FileSystem fileSystem, Path path) throws IOException {
+    String pathStr = path.toUri().getPath();
+    if (fileSystem instanceof DistributedFileSystem) {
+      return SHIMS.getFileId(fileSystem, pathStr);
+    }
+    // If we are not on DFS, we just hash the file name + size and hope for the best.
+    // TODO: we assume it only happens in tests. Fix?
+    int nameHash = pathStr.hashCode();
+    long fileSize = fileSystem.getFileStatus(path).getLen();
+    long id = ((fileSize ^ (fileSize >>> 32)) << 32) | ((long)nameHash & 0xffffffffL);
+    LOG.warn("Cannot get unique file ID from "
+        + fileSystem.getClass().getSimpleName() + "; using " + id + "(" + pathStr
+        + "," + nameHash + "," + fileSize + ")");
+    return id;
+  }
+
+  // TODO: this relies on HDFS not changing the format; we assume if we could get inode ID, this
+  //       is still going to work. Otherwise, file IDs can be turned off. Later, we should use
+  //       as public utility method in HDFS to obtain the inode-based path.
+  private static String HDFS_ID_PATH_PREFIX = "/.reserved/.inodes/";
+
+  public static Path getFileIdPath(
+      FileSystem fileSystem, Path path, long fileId) {
+    return (fileSystem instanceof DistributedFileSystem)
+        ? new Path(HDFS_ID_PATH_PREFIX + fileId) : path;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java
index 3d2525b..a5cead4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java
@@ -3,18 +3,20 @@ package org.apache.hadoop.hive.ql.io.orc;
 import java.io.IOException;
 import java.util.List;
 
-import org.apache.hadoop.hive.llap.Consumer;
-import org.apache.hadoop.hive.llap.io.api.EncodedColumnBatch;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
+import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl.OrcEncodedColumnBatch;
 import org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding;
 import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex;
 import org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream;
+import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
 
 public interface EncodedReader {
+  // TODO: document
   void readEncodedColumns(int stripeIx, StripeInformation stripe,
       RowIndex[] index, List<ColumnEncoding> encodings, List<Stream> streams,
-      boolean[] included, boolean[][] colRgs) throws IOException;
+      boolean[] included, boolean[][] colRgs,
+      Consumer<OrcEncodedColumnBatch> consumer) throws IOException;
 
   void close() throws IOException;
+
+  void setDebugTracing(boolean isEnabled);
 }
\ No newline at end of file