You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/07/27 22:48:44 UTC
[3/4] hive git commit: HIVE-11259 : LLAP: clean up ORC dependencies
part 1 (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
index c16be38..d51acb5 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
@@ -19,11 +19,10 @@ package org.apache.hadoop.hive.llap.io.decode;
import java.io.IOException;
-import org.apache.hadoop.hive.llap.Consumer;
+import org.apache.hadoop.hive.common.io.storage_api.EncodedColumnBatch;
+import org.apache.hadoop.hive.common.io.storage_api.EncodedColumnBatch.ColumnStreamData;
import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters;
-import org.apache.hadoop.hive.llap.io.api.EncodedColumnBatch;
import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics;
@@ -31,6 +30,8 @@ import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.io.orc.CompressionCodec;
import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl.OrcEncodedColumnBatch;
+import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
+import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
import org.apache.hadoop.hive.ql.io.orc.EncodedTreeReaderFactory;
import org.apache.hadoop.hive.ql.io.orc.OrcProto;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
@@ -71,14 +72,14 @@ public class OrcEncodedDataConsumer
protected void decodeBatch(OrcEncodedColumnBatch batch,
Consumer<ColumnVectorBatch> downstreamConsumer) {
long startTime = counters.startTimeCounter();
- int currentStripeIndex = batch.batchKey.stripeIx;
+ int currentStripeIndex = batch.getBatchKey().stripeIx;
boolean sameStripe = currentStripeIndex == previousStripeIndex;
try {
OrcStripeMetadata stripeMetadata = stripes[currentStripeIndex];
// Get non null row count from root column, to get max vector batches
- int rgIdx = batch.batchKey.rgIx;
+ int rgIdx = batch.getBatchKey().rgIx;
long nonNullRowCount = -1;
if (rgIdx == OrcEncodedColumnBatch.ALL_RGS) {
nonNullRowCount = stripeMetadata.getRowCount();
@@ -88,7 +89,7 @@ public class OrcEncodedDataConsumer
}
int maxBatchesRG = (int) ((nonNullRowCount / VectorizedRowBatch.DEFAULT_SIZE) + 1);
int batchSize = VectorizedRowBatch.DEFAULT_SIZE;
- int numCols = batch.columnIxs.length;
+ int numCols = batch.getColumnIxs().length;
if (columnReaders == null || !sameStripe) {
this.columnReaders = EncodedTreeReaderFactory.createEncodedTreeReader(numCols,
fileMetadata.getTypes(), stripeMetadata.getEncodings(), batch, codec, skipCorrupt);
@@ -106,10 +107,10 @@ public class OrcEncodedDataConsumer
}
ColumnVectorBatch cvb = cvbPool.take();
- assert cvb.cols.length == batch.columnIxs.length; // Must be constant per split.
+ assert cvb.cols.length == batch.getColumnIxs().length; // Must be constant per split.
cvb.size = batchSize;
- for (int idx = 0; idx < batch.columnIxs.length; idx++) {
+ for (int idx = 0; idx < batch.getColumnIxs().length; idx++) {
cvb.cols[idx] = (ColumnVector)columnReaders[idx].nextVector(cvb.cols[idx], batchSize);
}
@@ -130,8 +131,8 @@ public class OrcEncodedDataConsumer
EncodedColumnBatch<OrcBatchKey> batch, int numCols,
OrcStripeMetadata stripeMetadata) throws IOException {
for (int i = 0; i < numCols; i++) {
- int columnIndex = batch.columnIxs[i];
- int rowGroupIndex = batch.batchKey.rgIx;
+ int columnIndex = batch.getColumnIxs()[i];
+ int rowGroupIndex = batch.getBatchKey().rgIx;
OrcProto.RowIndex rowIndex = stripeMetadata.getRowIndexes()[columnIndex];
OrcProto.RowIndexEntry rowIndexEntry = rowIndex.getEntry(rowGroupIndex);
columnReaders[i].seek(new RecordReaderImpl.PositionProviderImpl(rowIndexEntry));
@@ -142,9 +143,9 @@ public class OrcEncodedDataConsumer
EncodedColumnBatch<OrcBatchKey> batch, boolean sameStripe, int numCols,
OrcStripeMetadata stripeMetadata) throws IOException {
for (int i = 0; i < numCols; i++) {
- int columnIndex = batch.columnIxs[i];
- int rowGroupIndex = batch.batchKey.rgIx;
- EncodedColumnBatch.StreamBuffer[] streamBuffers = batch.columnData[i];
+ int columnIndex = batch.getColumnIxs()[i];
+ int rowGroupIndex = batch.getBatchKey().rgIx;
+ ColumnStreamData[] streamBuffers = batch.getColumnData()[i];
OrcProto.RowIndex rowIndex = stripeMetadata.getRowIndexes()[columnIndex];
OrcProto.RowIndexEntry rowIndexEntry = rowIndex.getEntry(rowGroupIndex);
columnReaders[i].setBuffers(streamBuffers, sameStripe);
http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index 5cf0780..f88a943 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -1,6 +1,7 @@
package org.apache.hadoop.hive.llap.io.encoded;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -12,25 +13,29 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.CallableWithNdc;
+import org.apache.hadoop.hive.common.DiskRange;
+import org.apache.hadoop.hive.common.DiskRangeList;
+import org.apache.hadoop.hive.common.io.storage_api.Allocator;
+import org.apache.hadoop.hive.common.io.storage_api.DataCache;
+import org.apache.hadoop.hive.common.io.storage_api.DataReader;
+import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
+import org.apache.hadoop.hive.common.io.storage_api.EncodedColumnBatch.ColumnStreamData;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.Consumer;
import org.apache.hadoop.hive.llap.ConsumerFeedback;
import org.apache.hadoop.hive.llap.DebugUtils;
import org.apache.hadoop.hive.llap.cache.Cache;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters;
import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters.Counter;
-import org.apache.hadoop.hive.llap.io.api.EncodedColumnBatch.StreamBuffer;
-import org.apache.hadoop.hive.llap.io.api.cache.LlapMemoryBuffer;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache;
import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcCacheKey;
import org.apache.hadoop.hive.llap.io.decode.OrcEncodedDataConsumer;
import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
import org.apache.hadoop.hive.llap.io.metadata.OrcMetadataCache;
import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
import org.apache.hadoop.hive.ql.exec.DDLTask;
import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.io.HdfsUtils;
import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
import org.apache.hadoop.hive.ql.io.orc.EncodedReader;
import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl;
@@ -44,9 +49,14 @@ import org.apache.hadoop.hive.ql.io.orc.OrcSplit;
import org.apache.hadoop.hive.ql.io.orc.Reader;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.SargApplier;
+import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
+import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
+import org.apache.hadoop.hive.ql.io.orc.llap.OrcCacheKey;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderUtils;
import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
@@ -235,7 +245,9 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
try {
ensureOrcReader();
// Reader creating updates HDFS counters, don't do it here.
- stripeReader = orcReader.encodedReader(fileId, lowLevelCache, counters, dataConsumer);
+ DataWrapperForOrc dw = new DataWrapperForOrc();
+ stripeReader = orcReader.encodedReader(fileId, dw, dw);
+ stripeReader.setDebugTracing(DebugUtils.isTraceOrcEnabled());
} catch (Throwable t) {
consumer.setError(t);
recordReaderTime(startTime);
@@ -340,7 +352,8 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
// Also, currently readEncodedColumns is not stoppable. The consumer will discard the
// data it receives for one stripe. We could probably interrupt it, if it checked that.
stripeReader.readEncodedColumns(stripeIx, stripe, stripeMetadata.getRowIndexes(),
- stripeMetadata.getEncodings(), stripeMetadata.getStreams(), stripeIncludes, colRgs);
+ stripeMetadata.getEncodings(), stripeMetadata.getStreams(), stripeIncludes,
+ colRgs, dataConsumer);
} catch (Throwable t) {
consumer.setError(t);
cleanupReaders();
@@ -434,7 +447,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
}
}
LOG.warn("Split for " + split.getPath() + " (" + split.getClass() + ") does not have file ID");
- return RecordReaderUtils.getFileId(fs, split.getPath());
+ return HdfsUtils.getFileId(fs, split.getPath());
}
private boolean[][] genStripeColRgs(List<Integer> stripeCols, boolean[][] globalColRgs) {
@@ -499,7 +512,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
*/
private void ensureOrcReader() throws IOException {
if (orcReader != null) return;
- Path path = RecordReaderUtils.getFileIdPath(fs, split.getPath(), fileId);
+ Path path = HdfsUtils.getFileIdPath(fs, split.getPath(), fileId);
if (DebugUtils.isTraceOrcEnabled()) {
LOG.info("Creating reader for " + path + " (" + split.getPath() + ")");
}
@@ -578,15 +591,15 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
@Override
public void returnData(OrcEncodedColumnBatch ecb) {
- for (StreamBuffer[] datas : ecb.columnData) {
- for (StreamBuffer data : datas) {
+ for (ColumnStreamData[] datas : ecb.getColumnData()) {
+ for (ColumnStreamData data : datas) {
if (data.decRef() != 0) continue;
if (DebugUtils.isTraceLockingEnabled()) {
- for (LlapMemoryBuffer buf : data.cacheBuffers) {
+ for (MemoryBuffer buf : data.getCacheBuffers()) {
LlapIoImpl.LOG.info("Unlocking " + buf + " at the end of processing");
}
}
- lowLevelCache.releaseBuffers(data.cacheBuffers);
+ lowLevelCache.releaseBuffers(data.getCacheBuffers());
EncodedReaderImpl.SB_POOL.offer(data);
}
}
@@ -743,12 +756,12 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
if ((readMask == SargApplier.READ_NO_RGS) || (readMask != SargApplier.READ_ALL_RGS
&& (readMask.length <= rgIx || !readMask[rgIx]))) continue;
key.colIx = columnIds.get(colIxMod);
- StreamBuffer[] cached = cache.get(key);
+ ColumnStreamData[] cached = cache.get(key);
if (cached == null) {
isMissingAnyRgs[colIxMod] = true;
continue;
}
- col.setAllStreams(colIxMod, key.colIx, cached);
+ col.setAllStreamsData(colIxMod, key.colIx, cached);
hasAnyCached = true;
if (readMask == SargApplier.READ_ALL_RGS) {
// We were going to read all RGs, but some were in cache, allocate the mask.
@@ -793,16 +806,16 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
public void consumeData(OrcEncodedColumnBatch data) {
// Store object in cache; create new key object - cannot be reused.
assert cache != null;
- for (int i = 0; i < data.columnData.length; ++i) {
- OrcCacheKey key = new OrcCacheKey(data.batchKey, data.columnIxs[i]);
- StreamBuffer[] toCache = data.columnData[i];
- StreamBuffer[] cached = cache.cacheOrGet(key, toCache);
+ for (int i = 0; i < data.getColumnData().length; ++i) {
+ OrcCacheKey key = new OrcCacheKey(data.getBatchKey(), data.getColumnIxs()[i]);
+ ColumnStreamData[] toCache = data.getColumnData()[i];
+ ColumnStreamData[] cached = cache.cacheOrGet(key, toCache);
if (toCache != cached) {
- for (StreamBuffer sb : toCache) {
+ for (ColumnStreamData sb : toCache) {
if (sb.decRef() != 0) continue;
- lowLevelCache.releaseBuffers(sb.cacheBuffers);
+ lowLevelCache.releaseBuffers(sb.getCacheBuffers());
}
- data.columnData[i] = cached;
+ data.getColumnData()[i] = cached;
}
}
consumer.consumeData(data);
@@ -812,4 +825,82 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
public void setError(Throwable t) {
consumer.setError(t);
}
+
+ private class DataWrapperForOrc implements DataReader, DataCache {
+ private DataReader orcDataReader;
+
+ public DataWrapperForOrc() {
+ boolean useZeroCopy = (conf != null)
+ && HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ORC_ZEROCOPY);
+ if (useZeroCopy && !lowLevelCache.getAllocator().isDirectAlloc()) {
+ throw new UnsupportedOperationException("Cannot use zero-copy reader with non-direct cache "
+ + "buffers; either disable zero-copy or enable direct cache allocation");
+ }
+ this.orcDataReader = orcReader.createDefaultDataReader(useZeroCopy);
+ }
+
+ @Override
+ public DiskRangeList getFileData(long fileId, DiskRangeList range,
+ long baseOffset, DiskRangeListFactory factory, BooleanRef gotAllData) {
+ return lowLevelCache.getFileData(fileId, range, baseOffset, factory, counters, gotAllData);
+ }
+
+ @Override
+ public long[] putFileData(long fileId, DiskRange[] ranges,
+ MemoryBuffer[] data, long baseOffset) {
+ return lowLevelCache.putFileData(
+ fileId, ranges, data, baseOffset, Priority.NORMAL, counters);
+ }
+
+ @Override
+ public void releaseBuffer(MemoryBuffer buffer) {
+ lowLevelCache.releaseBuffer(buffer);
+ }
+
+ @Override
+ public void reuseBuffer(MemoryBuffer buffer) {
+ boolean isReused = lowLevelCache.reuseBuffer(buffer);
+ assert isReused;
+ }
+
+ @Override
+ public Allocator getAllocator() {
+ return lowLevelCache.getAllocator();
+ }
+
+ @Override
+ public void close() throws IOException {
+ orcDataReader.close();
+ }
+
+ @Override
+ public DiskRangeList readFileData(DiskRangeList range, long baseOffset,
+ boolean doForceDirect) throws IOException {
+ long startTime = counters.startTimeCounter();
+ DiskRangeList result = orcDataReader.readFileData(range, baseOffset, doForceDirect);
+ counters.recordHdfsTime(startTime);
+ if (DebugUtils.isTraceOrcEnabled() && LOG.isInfoEnabled()) {
+ LOG.info("Disk ranges after disk read (file " + fileId + ", base offset " + baseOffset
+ + "): " + RecordReaderUtils.stringifyDiskRanges(result));
+ }
+ return result;
+ }
+
+ @Override
+ public boolean isTrackingDiskRanges() {
+ return orcDataReader.isTrackingDiskRanges();
+ }
+
+ @Override
+ public void releaseBuffer(ByteBuffer buffer) {
+ orcDataReader.releaseBuffer(buffer);
+ }
+
+ @Override
+ public void open() throws IOException {
+ long startTime = counters.startTimeCounter();
+ orcDataReader.open();
+ counters.recordHdfsTime(startTime);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
index dbfe4df..635ba6c 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
@@ -21,13 +21,10 @@ package org.apache.hadoop.hive.llap.io.metadata;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.hadoop.hive.llap.cache.EvictionListener;
-import org.apache.hadoop.hive.llap.cache.LlapCacheableBuffer;
-import org.apache.hadoop.hive.llap.cache.LowLevelCacheMemoryManager;
import org.apache.hadoop.hive.llap.cache.LowLevelCachePolicy;
import org.apache.hadoop.hive.llap.cache.MemoryManager;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache.Priority;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
+import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
public class OrcMetadataCache {
private final ConcurrentHashMap<Long, OrcFileMetadata> metadata =
http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
index c0b8b34..762c9f9 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator;
import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator.ObjectEstimator;
import org.apache.hadoop.hive.llap.cache.EvictionDispatcher;
import org.apache.hadoop.hive.llap.cache.LlapCacheableBuffer;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
import org.apache.hadoop.hive.ql.io.orc.MetadataReader;
import org.apache.hadoop.hive.ql.io.orc.OrcProto.BloomFilter;
import org.apache.hadoop.hive.ql.io.orc.OrcProto.BloomFilterIndex;
@@ -37,6 +36,7 @@ import org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream;
import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
+import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
import com.google.common.annotations.VisibleForTesting;
http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
index 50d5e19..3299135 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
@@ -30,9 +30,9 @@ import java.util.concurrent.FutureTask;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
+import org.apache.hadoop.hive.common.io.storage_api.Allocator.AllocatorOutOfMemoryException;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.cache.Allocator.LlapCacheOutOfMemoryException;
-import org.apache.hadoop.hive.llap.io.api.cache.LlapMemoryBuffer;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
import org.junit.Test;
@@ -191,7 +191,7 @@ public class TestBuddyAllocator {
}
private void allocSameSize(BuddyAllocator a, int allocCount, int sizeLog2) throws Exception {
- LlapMemoryBuffer[][] allocs = new LlapMemoryBuffer[allocCount][];
+ MemoryBuffer[][] allocs = new MemoryBuffer[allocCount][];
long[][] testValues = new long[allocCount][];
for (int j = 0; j < allocCount; ++j) {
allocateAndUseBuffer(a, allocs, testValues, 1, j, sizeLog2);
@@ -202,7 +202,7 @@ public class TestBuddyAllocator {
private void allocateUp(BuddyAllocator a, int min, int max, int allocPerSize,
boolean isSameOrderDealloc) throws Exception {
int sizes = max - min + 1;
- LlapMemoryBuffer[][] allocs = new LlapMemoryBuffer[sizes][];
+ MemoryBuffer[][] allocs = new MemoryBuffer[sizes][];
// Put in the beginning; relies on the knowledge of internal implementation. Pave?
long[][] testValues = new long[sizes][];
for (int i = min; i <= max; ++i) {
@@ -214,7 +214,7 @@ public class TestBuddyAllocator {
private void allocateDown(BuddyAllocator a, int min, int max, int allocPerSize,
boolean isSameOrderDealloc) throws Exception {
int sizes = max - min + 1;
- LlapMemoryBuffer[][] allocs = new LlapMemoryBuffer[sizes][];
+ MemoryBuffer[][] allocs = new MemoryBuffer[sizes][];
// Put in the beginning; relies on the knowledge of internal implementation. Pave?
long[][] testValues = new long[sizes][];
for (int i = max; i >= min; --i) {
@@ -223,20 +223,20 @@ public class TestBuddyAllocator {
deallocUpOrDown(a, isSameOrderDealloc, allocs, testValues);
}
- private void allocateAndUseBuffer(BuddyAllocator a, LlapMemoryBuffer[][] allocs,
+ private void allocateAndUseBuffer(BuddyAllocator a, MemoryBuffer[][] allocs,
long[][] testValues, int allocCount, int index, int sizeLog2) throws Exception {
- allocs[index] = new LlapMemoryBuffer[allocCount];
+ allocs[index] = new MemoryBuffer[allocCount];
testValues[index] = new long[allocCount];
int size = (1 << sizeLog2) - 1;
try {
a.allocateMultiple(allocs[index], size);
- } catch (LlapCacheOutOfMemoryException ex) {
+ } catch (AllocatorOutOfMemoryException ex) {
LOG.error("Failed to allocate " + allocCount + " of " + size + "; " + a.debugDump());
throw ex;
}
// LOG.info("Allocated " + allocCount + " of " + size + "; " + a.debugDump());
for (int j = 0; j < allocCount; ++j) {
- LlapMemoryBuffer mem = allocs[index][j];
+ MemoryBuffer mem = allocs[index][j];
long testValue = testValues[index][j] = rdm.nextLong();
int pos = mem.getByteBufferRaw().position();
mem.getByteBufferRaw().putLong(pos, testValue);
@@ -248,7 +248,7 @@ public class TestBuddyAllocator {
}
private void deallocUpOrDown(BuddyAllocator a, boolean isSameOrderDealloc,
- LlapMemoryBuffer[][] allocs, long[][] testValues) {
+ MemoryBuffer[][] allocs, long[][] testValues) {
if (isSameOrderDealloc) {
for (int i = 0; i < allocs.length; ++i) {
deallocBuffers(a, allocs[i], testValues[i]);
@@ -261,7 +261,7 @@ public class TestBuddyAllocator {
}
private void deallocBuffers(
- BuddyAllocator a, LlapMemoryBuffer[] allocs, long[] testValues) {
+ BuddyAllocator a, MemoryBuffer[] allocs, long[] testValues) {
for (int j = 0; j < allocs.length; ++j) {
LlapDataBuffer mem = (LlapDataBuffer)allocs[j];
int pos = mem.getByteBufferRaw().position();
http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
index 4697445..382e948 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
@@ -30,7 +30,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator;
import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator.ObjectEstimator;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
import org.apache.hadoop.hive.ql.io.orc.MetadataReader;
@@ -54,6 +53,7 @@ import org.apache.hadoop.hive.ql.io.orc.OrcProto.TimestampStatistics;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.Index;
import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
+import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.junit.Test;
import org.mockito.Mockito;
http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
index 2c87ec1..d4b8a46 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
@@ -35,26 +35,28 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.DiskRange;
import org.apache.hadoop.hive.common.DiskRangeList;
-import org.apache.hadoop.hive.common.DiskRangeList.DiskRangeListCreateHelper;
-import org.apache.hadoop.hive.llap.io.api.cache.LlapMemoryBuffer;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache.CacheChunkFactory;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache.Priority;
+import org.apache.hadoop.hive.common.DiskRangeList.CreateHelper;
+import org.apache.hadoop.hive.common.io.storage_api.DataCache.DiskRangeListFactory;
+import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.CacheChunk;
+import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl.CacheChunk;
import org.junit.Test;
public class TestLowLevelCacheImpl {
private static final Log LOG = LogFactory.getLog(TestLowLevelCacheImpl.class);
- private static final CacheChunkFactory testFactory = new CacheChunkFactory() {
- public DiskRangeList createCacheChunk(LlapMemoryBuffer buffer, long offset, long end) {
- return new CacheChunk(buffer, offset, end);
+ private static final DiskRangeListFactory testFactory = new DiskRangeListFactory() {
+ public DiskRangeList createCacheChunk(MemoryBuffer buffer, long offset, long end) {
+ CacheChunk cc = new CacheChunk();
+ cc.init(buffer, offset, end);
+ return cc;
}
};
private static class DummyAllocator implements EvictionAwareAllocator {
@Override
- public void allocateMultiple(LlapMemoryBuffer[] dest, int size) {
+ public void allocateMultiple(MemoryBuffer[] dest, int size) {
for (int i = 0; i < dest.length; ++i) {
LlapDataBuffer buf = new LlapDataBuffer();
buf.initialize(0, null, -1, size);
@@ -63,11 +65,11 @@ public class TestLowLevelCacheImpl {
}
@Override
- public void deallocate(LlapMemoryBuffer buffer) {
+ public void deallocate(MemoryBuffer buffer) {
}
@Override
- public void deallocateEvicted(LlapMemoryBuffer buffer) {
+ public void deallocateEvicted(MemoryBuffer buffer) {
}
@Override
@@ -79,6 +81,11 @@ public class TestLowLevelCacheImpl {
public int getMaxAllocation() {
return 0;
}
+
+ @Override
+ public MemoryBuffer createUnallocated() {
+ return new LlapDataBuffer();
+ }
}
private static class DummyCachePolicy implements LowLevelCachePolicy {
@@ -115,16 +122,16 @@ public class TestLowLevelCacheImpl {
LlapDaemonCacheMetrics.create("test", "1"), new DummyCachePolicy(),
new DummyAllocator(), true, -1); // no cleanup thread
long fn1 = 1, fn2 = 2;
- LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[] { fb(), fb(), fb(), fb(), fb(), fb() };
+ MemoryBuffer[] fakes = new MemoryBuffer[] { fb(), fb(), fb(), fb(), fb(), fb() };
verifyRefcount(fakes, 1, 1, 1, 1, 1, 1);
- assertNull(cache.putFileData(fn1, drs(1, 2), fbs(fakes, 0, 1), 0, Priority.NORMAL));
- assertNull(cache.putFileData(fn2, drs(1, 2), fbs(fakes, 2, 3), 0, Priority.NORMAL));
+ assertNull(cache.putFileData(fn1, drs(1, 2), fbs(fakes, 0, 1), 0, Priority.NORMAL, null));
+ assertNull(cache.putFileData(fn2, drs(1, 2), fbs(fakes, 2, 3), 0, Priority.NORMAL, null));
verifyCacheGet(cache, fn1, 1, 3, fakes[0], fakes[1]);
verifyCacheGet(cache, fn2, 1, 3, fakes[2], fakes[3]);
verifyCacheGet(cache, fn1, 2, 4, fakes[1], dr(3, 4));
verifyRefcount(fakes, 3, 4, 3, 3, 1, 1);
- LlapMemoryBuffer[] bufsDiff = fbs(fakes, 4, 5);
- long[] mask = cache.putFileData(fn1, drs(3, 1), bufsDiff, 0, Priority.NORMAL);
+ MemoryBuffer[] bufsDiff = fbs(fakes, 4, 5);
+ long[] mask = cache.putFileData(fn1, drs(3, 1), bufsDiff, 0, Priority.NORMAL, null);
assertEquals(1, mask.length);
assertEquals(2, mask[0]); // 2nd bit set - element 2 was already in cache.
assertSame(fakes[0], bufsDiff[1]); // Should have been replaced
@@ -134,7 +141,7 @@ public class TestLowLevelCacheImpl {
}
private void verifyCacheGet(LowLevelCacheImpl cache, long fileId, Object... stuff) {
- DiskRangeListCreateHelper list = new DiskRangeListCreateHelper();
+ CreateHelper list = new CreateHelper();
DiskRangeList iter = null;
int intCount = 0, lastInt = -1;
int resultCount = stuff.length;
@@ -153,13 +160,13 @@ public class TestLowLevelCacheImpl {
} else if (intCount >= 0) {
assertTrue(intCount == 0);
intCount = -1;
- iter = cache.getFileData(fileId, list.get(), 0, testFactory);
+ iter = cache.getFileData(fileId, list.get(), 0, testFactory, null, null);
assertEquals(resultCount, iter.listSize());
}
assertTrue(iter != null);
- if (obj instanceof LlapMemoryBuffer) {
+ if (obj instanceof MemoryBuffer) {
assertTrue(iter instanceof CacheChunk);
- assertSame(obj, ((CacheChunk)iter).buffer);
+ assertSame(obj, ((CacheChunk)iter).getBuffer());
} else {
assertTrue(iter.equals(obj));
}
@@ -173,9 +180,9 @@ public class TestLowLevelCacheImpl {
LlapDaemonCacheMetrics.create("test", "1"), new DummyCachePolicy(),
new DummyAllocator(), true, -1); // no cleanup thread
long fn = 1;
- LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[] { fb(), fb() };
+ MemoryBuffer[] fakes = new MemoryBuffer[] { fb(), fb() };
assertNull(cache.putFileData(
- fn, new DiskRange[] { dr(2, 4), dr(6, 8) }, fakes, 0, Priority.NORMAL));
+ fn, new DiskRange[] { dr(2, 4), dr(6, 8) }, fakes, 0, Priority.NORMAL, null));
verifyCacheGet(cache, fn, 1, 9, dr(1, 2), fakes[0], dr(4, 6), fakes[1], dr(8, 9));
verifyCacheGet(cache, fn, 2, 8, fakes[0], dr(4, 6), fakes[1]);
verifyCacheGet(cache, fn, 1, 5, dr(1, 2), fakes[0], dr(4, 5));
@@ -192,9 +199,9 @@ public class TestLowLevelCacheImpl {
LlapDaemonCacheMetrics.create("test", "1"), new DummyCachePolicy(),
new DummyAllocator(), false, -1); // no cleanup thread
long fn = 1;
- LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[] { fb(), fb() };
+ MemoryBuffer[] fakes = new MemoryBuffer[] { fb(), fb() };
assertNull(cache.putFileData(
- fn, new DiskRange[] { dr(2, 4), dr(6, 8) }, fakes, 0, Priority.NORMAL));
+ fn, new DiskRange[] { dr(2, 4), dr(6, 8) }, fakes, 0, Priority.NORMAL, null));
// We expect cache requests from the middle here
verifyCacheGet(cache, fn, 3, 4, fakes[0]);
verifyCacheGet(cache, fn, 3, 7, fakes[0], dr(4, 6), fakes[1]);
@@ -206,9 +213,9 @@ public class TestLowLevelCacheImpl {
LlapDaemonCacheMetrics.create("test", "1"), new DummyCachePolicy(),
new DummyAllocator(), true, -1); // no cleanup thread
long fn1 = 1, fn2 = 2;
- LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[] { fb(), fb(), fb() };
- assertNull(cache.putFileData(fn1, drs(1, 2), fbs(fakes, 0, 1), 0, Priority.NORMAL));
- assertNull(cache.putFileData(fn2, drs(1), fbs(fakes, 2), 0, Priority.NORMAL));
+ MemoryBuffer[] fakes = new MemoryBuffer[] { fb(), fb(), fb() };
+ assertNull(cache.putFileData(fn1, drs(1, 2), fbs(fakes, 0, 1), 0, Priority.NORMAL, null));
+ assertNull(cache.putFileData(fn2, drs(1), fbs(fakes, 2), 0, Priority.NORMAL, null));
verifyCacheGet(cache, fn1, 1, 3, fakes[0], fakes[1]);
verifyCacheGet(cache, fn2, 1, 2, fakes[2]);
verifyRefcount(fakes, 3, 3, 3);
@@ -225,23 +232,23 @@ public class TestLowLevelCacheImpl {
LlapDaemonCacheMetrics.create("test", "1"), new DummyCachePolicy(),
new DummyAllocator(), true, -1); // no cleanup thread
long fn1 = 1, fn2 = 2;
- LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[] {
+ MemoryBuffer[] fakes = new MemoryBuffer[] {
fb(), fb(), fb(), fb(), fb(), fb(), fb(), fb(), fb() };
- assertNull(cache.putFileData(fn1, drs(1, 2, 3), fbs(fakes, 0, 1, 2), 0, Priority.NORMAL));
- assertNull(cache.putFileData(fn2, drs(1), fbs(fakes, 3), 0, Priority.NORMAL));
+ assertNull(cache.putFileData(fn1, drs(1, 2, 3), fbs(fakes, 0, 1, 2), 0, Priority.NORMAL, null));
+ assertNull(cache.putFileData(fn2, drs(1), fbs(fakes, 3), 0, Priority.NORMAL, null));
evict(cache, fakes[0]);
evict(cache, fakes[3]);
long[] mask = cache.putFileData(
- fn1, drs(1, 2, 3, 4), fbs(fakes, 4, 5, 6, 7), 0, Priority.NORMAL);
+ fn1, drs(1, 2, 3, 4), fbs(fakes, 4, 5, 6, 7), 0, Priority.NORMAL, null);
assertEquals(1, mask.length);
assertEquals(6, mask[0]); // Buffers at offset 2 & 3 exist; 1 exists and is stale; 4 doesn't
- assertNull(cache.putFileData(fn2, drs(1), fbs(fakes, 8), 0, Priority.NORMAL));
+ assertNull(cache.putFileData(fn2, drs(1), fbs(fakes, 8), 0, Priority.NORMAL, null));
verifyCacheGet(cache, fn1, 1, 5, fakes[4], fakes[1], fakes[2], fakes[7]);
}
@Test
public void testCacheMetrics() {
- DiskRangeListCreateHelper list = new DiskRangeListCreateHelper();
+ CreateHelper list = new CreateHelper();
list.addOrMerge(0, 100, true, false);
list.addOrMerge(100, 200, true, false);
list.addOrMerge(200, 300, true, false);
@@ -249,7 +256,7 @@ public class TestLowLevelCacheImpl {
list.addOrMerge(400, 500, true, false);
assertEquals(1, list.get().listSize());
assertEquals(500, list.get().getTotalLength());
- list = new DiskRangeListCreateHelper();
+ list = new CreateHelper();
list.addOrMerge(0, 100, false, false);
list.addOrMerge(100, 200, false, false);
list.addOrMerge(200, 300, false, false);
@@ -257,7 +264,7 @@ public class TestLowLevelCacheImpl {
list.addOrMerge(400, 500, false, false);
assertEquals(5, list.get().listSize());
assertEquals(500, list.get().getTotalLength());
- list = new DiskRangeListCreateHelper();
+ list = new CreateHelper();
list.addOrMerge(0, 100, true, false);
list.addOrMerge(100, 200, true, false);
list.addOrMerge(200, 300, false, false);
@@ -270,35 +277,35 @@ public class TestLowLevelCacheImpl {
LowLevelCacheImpl cache = new LowLevelCacheImpl(metrics,
new DummyCachePolicy(), new DummyAllocator(), true, -1); // no cleanup thread
long fn = 1;
- LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[]{fb(), fb(), fb()};
+ MemoryBuffer[] fakes = new MemoryBuffer[]{fb(), fb(), fb()};
cache.putFileData(fn, new DiskRange[]{dr(0, 100), dr(300, 500), dr(800, 1000)},
- fakes, 0, Priority.NORMAL);
+ fakes, 0, Priority.NORMAL, null);
assertEquals(0, metrics.getCacheRequestedBytes());
assertEquals(0, metrics.getCacheHitBytes());
- list = new DiskRangeListCreateHelper();
+ list = new CreateHelper();
list.addOrMerge(0, 1000, true, false);
- cache.getFileData(fn, list.get(), 0, testFactory);
+ cache.getFileData(fn, list.get(), 0, testFactory, null, null);
assertEquals(1000, metrics.getCacheRequestedBytes());
assertEquals(500, metrics.getCacheHitBytes());
- list = new DiskRangeListCreateHelper();
+ list = new CreateHelper();
list.addOrMerge(0, 100, true, false);
- cache.getFileData(fn, list.get(), 0, testFactory);
+ cache.getFileData(fn, list.get(), 0, testFactory, null, null);
assertEquals(1100, metrics.getCacheRequestedBytes());
assertEquals(600, metrics.getCacheHitBytes());
- list = new DiskRangeListCreateHelper();
+ list = new CreateHelper();
list.addOrMerge(0, 100, true, false);
list.addOrMerge(300, 500, true, false);
list.addOrMerge(800, 1000, true, false);
- cache.getFileData(fn, list.get(), 0, testFactory);
+ cache.getFileData(fn, list.get(), 0, testFactory, null, null);
assertEquals(1600, metrics.getCacheRequestedBytes());
assertEquals(1100, metrics.getCacheHitBytes());
- list = new DiskRangeListCreateHelper();
+ list = new CreateHelper();
list.addOrMerge(300, 500, true, false);
list.addOrMerge(1000, 2000, true, false);
- cache.getFileData(fn, list.get(), 0, testFactory);
+ cache.getFileData(fn, list.get(), 0, testFactory, null, null);
assertEquals(2800, metrics.getCacheRequestedBytes());
assertEquals(1300, metrics.getCacheHitBytes());
}
@@ -325,12 +332,12 @@ public class TestLowLevelCacheImpl {
if (isGet) {
int[] offsets = new int[count];
count = generateOffsets(offsetsToUse, rdm, offsets);
- DiskRangeListCreateHelper list = new DiskRangeListCreateHelper();
+ CreateHelper list = new CreateHelper();
for (int j = 0; i < count; ++i) {
list.addOrMerge(offsets[j], offsets[j] + 1, true, false);
}
- DiskRangeList iter = cache.getFileData(fileName, list.get(), 0, testFactory);
+ DiskRangeList iter = cache.getFileData(fileName, list.get(), 0, testFactory, null, null);
int j = -1;
while (iter != null) {
++j;
@@ -339,7 +346,7 @@ public class TestLowLevelCacheImpl {
continue;
}
++gets;
- LlapDataBuffer result = (LlapDataBuffer)((CacheChunk)iter).buffer;
+ LlapDataBuffer result = (LlapDataBuffer)((CacheChunk)iter).getBuffer();
assertEquals(makeFakeArenaIndex(fileIndex, offsets[j]), result.arenaIndex);
cache.releaseBuffer(result);
iter = iter.next;
@@ -352,13 +359,13 @@ public class TestLowLevelCacheImpl {
ranges[j] = dr(next, next + 1);
offsets[j] = next;
}
- LlapMemoryBuffer[] buffers = new LlapMemoryBuffer[count];
+ MemoryBuffer[] buffers = new MemoryBuffer[count];
for (int j = 0; j < offsets.length; ++j) {
LlapDataBuffer buf = LowLevelCacheImpl.allocateFake();
buf.arenaIndex = makeFakeArenaIndex(fileIndex, offsets[j]);
buffers[j] = buf;
}
- long[] mask = cache.putFileData(fileName, ranges, buffers, 0, Priority.NORMAL);
+ long[] mask = cache.putFileData(fileName, ranges, buffers, 0, Priority.NORMAL, null);
puts += buffers.length;
long maskVal = 0;
if (mask != null) {
@@ -396,14 +403,14 @@ public class TestLowLevelCacheImpl {
DiskRangeList head = new DiskRangeList(0, offsetsToUse + 1);
isFirstFile = !isFirstFile;
long fileId = isFirstFile ? fn1 : fn2;
- head = cache.getFileData(fileId, head, 0, testFactory);
+ head = cache.getFileData(fileId, head, 0, testFactory, null, null);
DiskRange[] results = head.listToArray();
int startIndex = rdm.nextInt(results.length), index = startIndex;
LlapDataBuffer victim = null;
do {
DiskRange r = results[index];
if (r instanceof CacheChunk) {
- LlapDataBuffer result = (LlapDataBuffer)((CacheChunk)r).buffer;
+ LlapDataBuffer result = (LlapDataBuffer)((CacheChunk)r).getBuffer();
cache.releaseBuffer(result);
if (victim == null && result.invalidate()) {
++evictions;
@@ -452,7 +459,7 @@ public class TestLowLevelCacheImpl {
}
}
- private void evict(LowLevelCacheImpl cache, LlapMemoryBuffer fake) {
+ private void evict(LowLevelCacheImpl cache, MemoryBuffer fake) {
LlapDataBuffer victimBuffer = (LlapDataBuffer)fake;
int refCount = victimBuffer.getRefCount();
for (int i = 0; i < refCount; ++i) {
@@ -462,14 +469,14 @@ public class TestLowLevelCacheImpl {
cache.notifyEvicted(victimBuffer);
}
- private void verifyRefcount(LlapMemoryBuffer[] fakes, int... refCounts) {
+ private void verifyRefcount(MemoryBuffer[] fakes, int... refCounts) {
for (int i = 0; i < refCounts.length; ++i) {
assertEquals("At " + i, refCounts[i], ((LlapDataBuffer)fakes[i]).getRefCount());
}
}
- private LlapMemoryBuffer[] fbs(LlapMemoryBuffer[] fakes, int... indexes) {
- LlapMemoryBuffer[] rv = new LlapMemoryBuffer[indexes.length];
+ private MemoryBuffer[] fbs(MemoryBuffer[] fakes, int... indexes) {
+ MemoryBuffer[] rv = new MemoryBuffer[indexes.length];
for (int i = 0; i < indexes.length; ++i) {
rv[i] = (indexes[i] == -1) ? null : fakes[indexes[i]];
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
index 7349208..a423eeb 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
@@ -33,7 +33,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache.Priority;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
import org.junit.Assume;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
index 3549fd9..2886d54 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache.Priority;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
import org.apache.hadoop.hive.llap.io.metadata.OrcMetadataCache;
import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java b/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
index 8b61fe4..8dc62d8 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
@@ -35,10 +35,6 @@ public class DebugUtils {
return isTraceOrcEnabled; // TODO: temporary, should be hardcoded false
}
- public static boolean isTraceRangesEnabled() {
- return false;
- }
-
public static boolean isTraceLockingEnabled() {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
new file mode 100644
index 0000000..0095d31
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
+
+public class HdfsUtils {
+ private static final HadoopShims SHIMS = ShimLoader.getHadoopShims();
+ private static final Log LOG = LogFactory.getLog(HdfsUtils.class);
+
+ public static long getFileId(FileSystem fileSystem, Path path) throws IOException {
+ String pathStr = path.toUri().getPath();
+ if (fileSystem instanceof DistributedFileSystem) {
+ return SHIMS.getFileId(fileSystem, pathStr);
+ }
+ // If we are not on DFS, we just hash the file name + size and hope for the best.
+ // TODO: we assume it only happens in tests. Fix?
+ int nameHash = pathStr.hashCode();
+ long fileSize = fileSystem.getFileStatus(path).getLen();
+ long id = ((fileSize ^ (fileSize >>> 32)) << 32) | ((long)nameHash & 0xffffffffL);
+ LOG.warn("Cannot get unique file ID from "
+ + fileSystem.getClass().getSimpleName() + "; using " + id + "(" + pathStr
+ + "," + nameHash + "," + fileSize + ")");
+ return id;
+ }
+
+ // TODO: this relies on HDFS not changing the format; we assume if we could get inode ID, this
+ // is still going to work. Otherwise, file IDs can be turned off. Later, we should use
+ // as public utility method in HDFS to obtain the inode-based path.
+ private static String HDFS_ID_PATH_PREFIX = "/.reserved/.inodes/";
+
+ public static Path getFileIdPath(
+ FileSystem fileSystem, Path path, long fileId) {
+ return (fileSystem instanceof DistributedFileSystem)
+ ? new Path(HDFS_ID_PATH_PREFIX + fileId) : path;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/1e3b59d3/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java
index 3d2525b..a5cead4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java
@@ -3,18 +3,20 @@ package org.apache.hadoop.hive.ql.io.orc;
import java.io.IOException;
import java.util.List;
-import org.apache.hadoop.hive.llap.Consumer;
-import org.apache.hadoop.hive.llap.io.api.EncodedColumnBatch;
-import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache;
-import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
+import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl.OrcEncodedColumnBatch;
import org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding;
import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex;
import org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream;
+import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
public interface EncodedReader {
+ // TODO: document
void readEncodedColumns(int stripeIx, StripeInformation stripe,
RowIndex[] index, List<ColumnEncoding> encodings, List<Stream> streams,
- boolean[] included, boolean[][] colRgs) throws IOException;
+ boolean[] included, boolean[][] colRgs,
+ Consumer<OrcEncodedColumnBatch> consumer) throws IOException;
void close() throws IOException;
+
+ void setDebugTracing(boolean isEnabled);
}
\ No newline at end of file