You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/08/11 22:46:59 UTC
[4/4] hive git commit: HIVE-11385 : LLAP: clean up ORC dependencies -
move encoded reader path into a separate package and reader (Sergey Shelukhin,
reviewed by Prasanth Jayachandran)
HIVE-11385 : LLAP: clean up ORC dependencies - move encoded reader path into a separate package and reader (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2374cfb7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2374cfb7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2374cfb7
Branch: refs/heads/llap
Commit: 2374cfb7ad67452c3380144c919ee1b2a0ee4f48
Parents: e563a1d
Author: Sergey Shelukhin <se...@apache.org>
Authored: Tue Aug 11 13:46:59 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Tue Aug 11 13:46:59 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/common/DiskRange.java | 102 --
.../hadoop/hive/common/DiskRangeInfo.java | 2 +
.../hadoop/hive/common/DiskRangeList.java | 212 ---
.../hive/common/io/storage_api/Allocator.java | 51 -
.../hive/common/io/storage_api/DataCache.java | 101 --
.../hive/common/io/storage_api/DataReader.java | 60 -
.../io/storage_api/EncodedColumnBatch.java | 139 --
.../common/io/storage_api/MemoryBuffer.java | 28 -
.../hadoop/hive/llap/cache/BuddyAllocator.java | 2 +-
.../apache/hadoop/hive/llap/cache/Cache.java | 2 +-
.../hive/llap/cache/EvictionAwareAllocator.java | 4 +-
.../hadoop/hive/llap/cache/LlapDataBuffer.java | 2 +-
.../hadoop/hive/llap/cache/LowLevelCache.java | 12 +-
.../hive/llap/cache/LowLevelCacheImpl.java | 14 +-
.../hadoop/hive/llap/cache/NoopCache.java | 2 +-
.../hive/llap/io/api/impl/LlapInputFormat.java | 2 +-
.../hive/llap/io/api/impl/LlapIoImpl.java | 3 +-
.../llap/io/decode/ColumnVectorProducer.java | 2 +-
.../llap/io/decode/EncodedDataConsumer.java | 4 +-
.../llap/io/decode/OrcColumnVectorProducer.java | 4 +-
.../llap/io/decode/OrcEncodedDataConsumer.java | 10 +-
.../llap/io/encoded/OrcEncodedDataReader.java | 33 +-
.../hive/llap/io/metadata/OrcMetadataCache.java | 2 +-
.../llap/io/metadata/OrcStripeMetadata.java | 2 +-
.../hive/llap/cache/TestBuddyAllocator.java | 4 +-
.../TestIncrementalObjectSizeEstimator.java | 2 +-
.../hive/llap/cache/TestLowLevelCacheImpl.java | 12 +-
.../org/apache/hadoop/hive/llap/DebugUtils.java | 2 +-
.../hadoop/hive/ql/io/orc/DataReader.java | 58 +
.../hadoop/hive/ql/io/orc/EncodedReader.java | 22 -
.../hive/ql/io/orc/EncodedReaderImpl.java | 1404 ------------------
.../ql/io/orc/EncodedTreeReaderFactory.java | 6 +-
.../apache/hadoop/hive/ql/io/orc/InStream.java | 2 +-
.../hive/ql/io/orc/MetadataReaderImpl.java | 2 +-
.../apache/hadoop/hive/ql/io/orc/OrcFile.java | 2 +
.../apache/hadoop/hive/ql/io/orc/OutStream.java | 4 +-
.../apache/hadoop/hive/ql/io/orc/Reader.java | 5 -
.../hadoop/hive/ql/io/orc/ReaderImpl.java | 16 +-
.../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 11 +-
.../hive/ql/io/orc/RecordReaderUtils.java | 19 +-
.../ql/io/orc/SettableUncompressedStream.java | 2 +-
.../hadoop/hive/ql/io/orc/StreamName.java | 2 +-
.../hadoop/hive/ql/io/orc/StreamUtils.java | 4 +-
.../hive/ql/io/orc/TreeReaderFactory.java | 2 +-
.../hadoop/hive/ql/io/orc/encoded/Consumer.java | 30 +
.../hive/ql/io/orc/encoded/EncodedOrcFile.java | 30 +
.../hive/ql/io/orc/encoded/EncodedReader.java | 39 +
.../ql/io/orc/encoded/EncodedReaderImpl.java | 1404 ++++++++++++++++++
.../hive/ql/io/orc/encoded/OrcBatchKey.java | 60 +
.../hive/ql/io/orc/encoded/OrcCacheKey.java | 58 +
.../hadoop/hive/ql/io/orc/encoded/Reader.java | 32 +
.../hive/ql/io/orc/encoded/ReaderImpl.java | 42 +
.../hadoop/hive/ql/io/orc/llap/Consumer.java | 30 -
.../hadoop/hive/ql/io/orc/llap/OrcBatchKey.java | 60 -
.../hadoop/hive/ql/io/orc/llap/OrcCacheKey.java | 58 -
.../hive/ql/io/orc/TestRecordReaderImpl.java | 2 +-
.../apache/hadoop/hive/common/io/Allocator.java | 53 +
.../apache/hadoop/hive/common/io/DataCache.java | 100 ++
.../apache/hadoop/hive/common/io/DiskRange.java | 102 ++
.../hadoop/hive/common/io/DiskRangeList.java | 212 +++
.../common/io/encoded/EncodedColumnBatch.java | 140 ++
.../hive/common/io/encoded/MemoryBuffer.java | 28 +
62 files changed, 2483 insertions(+), 2373 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/common/src/java/org/apache/hadoop/hive/common/DiskRange.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/DiskRange.java b/common/src/java/org/apache/hadoop/hive/common/DiskRange.java
deleted file mode 100644
index 1c15526..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/DiskRange.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common;
-
-import java.nio.ByteBuffer;
-
-/**
- * The sections of a file.
- */
-public class DiskRange {
- /** The first address. */
- protected long offset;
- /** The address afterwards. */
- protected long end;
-
- public DiskRange(long offset, long end) {
- this.offset = offset;
- this.end = end;
- if (end < offset) {
- throw new IllegalArgumentException("invalid range " + this);
- }
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null || other.getClass() != getClass()) {
- return false;
- }
- return equalRange((DiskRange) other);
- }
-
- public boolean equalRange(DiskRange other) {
- return other.offset == offset && other.end == end;
- }
-
- @Override
- public int hashCode() {
- return (int)(offset ^ (offset >>> 32)) * 31 + (int)(end ^ (end >>> 32));
- }
-
- @Override
- public String toString() {
- return "range start: " + offset + " end: " + end;
- }
-
- public long getOffset() {
- return offset;
- }
-
- public long getEnd() {
- return end;
- }
-
- public int getLength() {
- long len = this.end - this.offset;
- assert len <= Integer.MAX_VALUE;
- return (int)len;
- }
-
- // For subclasses
- public boolean hasData() {
- return false;
- }
-
- public DiskRange sliceAndShift(long offset, long end, long shiftBy) {
- // Rather, unexpected usage exception.
- throw new UnsupportedOperationException();
- }
-
- public ByteBuffer getData() {
- throw new UnsupportedOperationException();
- }
-
- protected boolean merge(long otherOffset, long otherEnd) {
- if (!overlap(offset, end, otherOffset, otherEnd)) return false;
- offset = Math.min(offset, otherOffset);
- end = Math.max(end, otherEnd);
- return true;
- }
-
- private static boolean overlap(long leftA, long rightA, long leftB, long rightB) {
- if (leftA <= leftB) {
- return rightA >= leftB;
- }
- return rightB >= leftA;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/common/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java b/common/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
index fb9a7b5..86b838c 100644
--- a/common/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
+++ b/common/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hive.common;
import java.util.List;
+import org.apache.hadoop.hive.common.io.DiskRange;
+
import com.google.common.collect.Lists;
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/common/src/java/org/apache/hadoop/hive/common/DiskRangeList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/DiskRangeList.java b/common/src/java/org/apache/hadoop/hive/common/DiskRangeList.java
deleted file mode 100644
index 4fa72a2..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/DiskRangeList.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/** Java linked list iterator interface is convoluted, and moreover concurrent modifications
- * of the same list by multiple iterators are impossible. Hence, this.
- * Java also doesn't support multiple inheritance, so this cannot be done as "aspect"... */
-public class DiskRangeList extends DiskRange {
- private static final Log LOG = LogFactory.getLog(DiskRangeList.class);
- public DiskRangeList prev, next;
-
- public DiskRangeList(long offset, long end) {
- super(offset, end);
- }
-
- /** Replaces this element with another in the list; returns the new element. */
- public DiskRangeList replaceSelfWith(DiskRangeList other) {
- other.prev = this.prev;
- other.next = this.next;
- if (this.prev != null) {
- this.prev.next = other;
- }
- if (this.next != null) {
- this.next.prev = other;
- }
- this.next = this.prev = null;
- return other;
- }
-
- /**
- * Inserts an intersecting range before current in the list and adjusts offset accordingly.
- * @returns the new element.
- */
- public DiskRangeList insertPartBefore(DiskRangeList other) {
- assert other.end >= this.offset;
- this.offset = other.end;
- other.prev = this.prev;
- other.next = this;
- if (this.prev != null) {
- this.prev.next = other;
- }
- this.prev = other;
- return other;
- }
-
- /**
- * Inserts an element after current in the list.
- * @returns the new element.
- * */
- public DiskRangeList insertAfter(DiskRangeList other) {
- other.next = this.next;
- other.prev = this;
- if (this.next != null) {
- this.next.prev = other;
- }
- this.next = other;
- return other;
- }
-
- /**
- * Inserts an intersecting range after current in the list and adjusts offset accordingly.
- * @returns the new element.
- */
- public DiskRangeList insertPartAfter(DiskRangeList other) {
- assert other.offset <= this.end;
- this.end = other.offset;
- return insertAfter(other);
- }
-
- /** Removes an element after current from the list. */
- public void removeAfter() {
- DiskRangeList other = this.next;
- this.next = other.next;
- if (this.next != null) {
- this.next.prev = this;
- }
- other.next = other.prev = null;
- }
-
- /** Removes the current element from the list. */
- public void removeSelf() {
- if (this.prev != null) {
- this.prev.next = this.next;
- }
- if (this.next != null) {
- this.next.prev = this.prev;
- }
- this.next = this.prev = null;
- }
-
- /** Splits current element in the list, using DiskRange::slice */
- public final DiskRangeList split(long cOffset) {
- insertAfter((DiskRangeList)this.sliceAndShift(cOffset, end, 0));
- return replaceSelfWith((DiskRangeList)this.sliceAndShift(offset, cOffset, 0));
- }
-
- public boolean hasContiguousNext() {
- return next != null && end == next.offset;
- }
-
- @VisibleForTesting
- public int listSize() {
- int result = 1;
- DiskRangeList current = this.next;
- while (current != null) {
- ++result;
- current = current.next;
- }
- return result;
- }
-
- public long getTotalLength() {
- long totalLength = getLength();
- DiskRangeList current = next;
- while (current != null) {
- totalLength += current.getLength();
- current = current.next;
- }
- return totalLength;
- }
-
- @VisibleForTesting
- public DiskRangeList[] listToArray() {
- DiskRangeList[] result = new DiskRangeList[listSize()];
- int i = 0;
- DiskRangeList current = this.next;
- while (current != null) {
- result[i] = current;
- ++i;
- current = current.next;
- }
- return result;
- }
-
- public static class CreateHelper {
- private DiskRangeList tail = null, head;
-
- public DiskRangeList getTail() {
- return tail;
- }
-
- public void addOrMerge(long offset, long end, boolean doMerge, boolean doLogNew) {
- if (doMerge && tail != null && tail.merge(offset, end)) return;
- if (doLogNew) {
- LOG.info("Creating new range; last range (which can include some previous adds) was "
- + tail);
- }
- DiskRangeList node = new DiskRangeList(offset, end);
- if (tail == null) {
- head = tail = node;
- } else {
- tail = tail.insertAfter(node);
- }
- }
-
- public DiskRangeList get() {
- return head;
- }
-
- public DiskRangeList extract() {
- DiskRangeList result = head;
- head = null;
- return result;
- }
- }
-
- /**
- * List in-place mutation helper - a bogus first element that is inserted before list head,
- * and thus remains constant even if head is replaced with some new range via in-place list
- * mutation. extract() can be used to obtain the modified list.
- */
- public static class MutateHelper extends DiskRangeList {
- public MutateHelper(DiskRangeList head) {
- super(-1, -1);
- assert head != null;
- assert head.prev == null;
- this.next = head;
- head.prev = this;
- }
-
- public DiskRangeList get() {
- return next;
- }
-
- public DiskRangeList extract() {
- DiskRangeList result = this.next;
- assert result != null;
- this.next = result.prev = null;
- return result;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/common/src/java/org/apache/hadoop/hive/common/io/storage_api/Allocator.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/storage_api/Allocator.java b/common/src/java/org/apache/hadoop/hive/common/io/storage_api/Allocator.java
deleted file mode 100644
index 0814fe7..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/io/storage_api/Allocator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common.io.storage_api;
-
-/** An allocator provided externally to storage classes to allocate MemoryBuffer-s. */
-public interface Allocator {
- public static class AllocatorOutOfMemoryException extends RuntimeException {
- public AllocatorOutOfMemoryException(String msg) {
- super(msg);
- }
-
- private static final long serialVersionUID = 268124648177151761L;
- }
-
- /**
- * Allocates multiple buffers of a given size.
- * @param dest Array where buffers are placed. Objects are reused if already there
- * (see createUnallocated), created otherwise.
- * @param size Allocation size.
- * @throws AllocatorOutOfMemoryException Cannot allocate.
- */
- void allocateMultiple(MemoryBuffer[] dest, int size) throws AllocatorOutOfMemoryException;
-
- /**
- * Creates an unallocated memory buffer object. This object can be passed to allocateMultiple
- * to allocate; this is useful if data structures are created for separate buffers that can
- * later be allocated together.
- */
- MemoryBuffer createUnallocated();
- /** Deallocates a memory buffer. */
- void deallocate(MemoryBuffer buffer);
- /** Whether the allocator uses direct buffers. */
- boolean isDirectAlloc();
- /** Maximum allocation size supported by this allocator. */
- int getMaxAllocation();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/common/src/java/org/apache/hadoop/hive/common/io/storage_api/DataCache.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/storage_api/DataCache.java b/common/src/java/org/apache/hadoop/hive/common/io/storage_api/DataCache.java
deleted file mode 100644
index 0ec67ea..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/io/storage_api/DataCache.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.common.io.storage_api;
-
-import org.apache.hadoop.hive.common.DiskRange;
-import org.apache.hadoop.hive.common.DiskRangeList;
-
-/** An abstract data cache that IO formats can use to retrieve and cache data. */
-public interface DataCache {
- public static final class BooleanRef {
- public boolean value;
- }
-
- /** Disk range factory used during cache retrieval. */
- public interface DiskRangeListFactory {
- DiskRangeList createCacheChunk(MemoryBuffer buffer, long startOffset, long endOffset);
- }
-
- /**
- * Gets file data for particular offsets. The range list is modified in place; it is then
- * returned (since the list head could have changed). Ranges are replaced with cached ranges.
- *
- * Any such buffer is locked in cache to prevent eviction, and must therefore be released
- * back to cache via a corresponding call (releaseBuffer) when the caller is done with it.
- *
- * In case of partial overlap with cached data, full cache blocks are always returned;
- * there's no capacity for partial matches in return type. The rules are as follows:
- * 1) If the requested range starts in the middle of a cached range, that cached range will not
- * be returned by default (e.g. if [100,200) and [200,300) are cached, the request for
- * [150,300) will only return [200,300) from cache). This may be configurable in impls.
- * This is because we assume well-known range start offsets are used (rg/stripe offsets), so
- * a request from the middle of the start doesn't make sense.
- * 2) If the requested range ends in the middle of a cached range, that entire cached range will
- * be returned (e.g. if [100,200) and [200,300) are cached, the request for [100,250) will
- * return both ranges). It should really be same as #1, however currently ORC uses estimated
- * end offsets; if we don't return the end block, the caller may read it from disk needlessly.
- *
- * @param fileId Unique ID of the target file on the file system.
- * @param range A set of DiskRange-s (linked list) that is to be retrieved. May be modified.
- * @param baseOffset base offset for the ranges (stripe/stream offset in case of ORC).
- * @param factory A factory to produce DiskRangeList-s out of cached MemoryBuffer-s.
- * @param gotAllData An out param - whether all the requested data was found in cache.
- * @return The new or modified list of DiskRange-s, where some ranges may contain cached data.
- */
- DiskRangeList getFileData(long fileId, DiskRangeList range, long baseOffset,
- DiskRangeListFactory factory, BooleanRef gotAllData);
-
- /**
- * Puts file data into cache, or gets older data in case of collisions.
- *
- * The memory buffers provided MUST be allocated via an allocator returned by getAllocator
- * method, to allow cache implementations that evict and then de-allocate the buffer.
- *
- * It is assumed that the caller will use the data immediately, therefore any buffers provided
- * to putFileData (or returned due to cache collision) are locked in cache to prevent eviction,
- * and must therefore be released back to cache via a corresponding call (releaseBuffer) when the
- * caller is done with it. Buffers rejected due to conflict will neither be locked, nor
- * automatically deallocated. The caller must take care to discard these buffers.
- *
- * @param fileId Unique ID of the target file on the file system.
- * @param ranges The ranges for which the data is being cached. These objects will not be stored.
- * @param data The data for the corresponding ranges.
- * @param baseOffset base offset for the ranges (stripe/stream offset in case of ORC).
- * @return null if all data was put; bitmask indicating which chunks were not put otherwise;
- * the replacement chunks from cache are updated directly in the array.
- */
- long[] putFileData(long fileId, DiskRange[] ranges, MemoryBuffer[] data, long baseOffset);
-
- /**
- * Releases the buffer returned by getFileData/provided to putFileData back to cache.
- * See respective javadocs for details.
- */
- void releaseBuffer(MemoryBuffer buffer);
-
- /**
- * Notifies the cache that the buffer returned from getFileData/provided to putFileData will
- * be used by another consumer and therefore released multiple times (one more time per call).
- */
- void reuseBuffer(MemoryBuffer buffer);
-
- /**
- * Gets the allocator associated with this DataCache.
- */
- Allocator getAllocator();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/common/src/java/org/apache/hadoop/hive/common/io/storage_api/DataReader.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/storage_api/DataReader.java b/common/src/java/org/apache/hadoop/hive/common/io/storage_api/DataReader.java
deleted file mode 100644
index 0e11e4e..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/io/storage_api/DataReader.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.common.io.storage_api;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hadoop.hive.common.DiskRangeList;
-import org.apache.hadoop.hive.common.io.storage_api.DataCache.DiskRangeListFactory;
-
-/** An abstract data reader that IO formats can use to read bytes from underlying storage. */
-public interface DataReader {
-
- /** Opens the DataReader, making it ready to use. */
- void open() throws IOException;
-
- /** Closes the DataReader. */
- void close() throws IOException;
-
- /** Reads the data.
- *
- * Note that for the cases such as zero-copy read, caller must release the disk ranges
- * produced after being done with them. Call isTrackingDiskRanges to find out if this is needed.
- * @param range List if disk ranges to read. Ranges with data will be ignored.
- * @param baseOffset Base offset from the start of the file of the ranges in disk range list.
- * @param doForceDirect Whether the data should be read into direct buffers.
- * @return New or modified list of DiskRange-s, where all the ranges are filled with data.
- */
- DiskRangeList readFileData(
- DiskRangeList range, long baseOffset, boolean doForceDirect) throws IOException;
-
-
- /**
- * Whether the user should release buffers created by readFileData. See readFileData javadoc.
- */
- boolean isTrackingDiskRanges();
-
- /**
- * Releases buffers created by readFileData. See readFileData javadoc.
- * @param toRelease The buffer to release.
- */
- void releaseBuffer(ByteBuffer toRelease);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/common/src/java/org/apache/hadoop/hive/common/io/storage_api/EncodedColumnBatch.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/storage_api/EncodedColumnBatch.java b/common/src/java/org/apache/hadoop/hive/common/io/storage_api/EncodedColumnBatch.java
deleted file mode 100644
index d51e3b4..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/io/storage_api/EncodedColumnBatch.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.common.io.storage_api;
-
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * A block of data for a given section of a file, similar to VRB but in encoded form.
- * Stores a set of buffers for each encoded stream that is a part of each column.
- */
-public class EncodedColumnBatch<BatchKey> {
- /**
- * Slice of the data for a stream for some column, stored inside MemoryBuffer's.
- * ColumnStreamData can be reused for many EncodedColumnBatch-es (e.g. dictionary stream), so
- * it tracks the number of such users via a refcount.
- */
- public static class ColumnStreamData {
- private List<MemoryBuffer> cacheBuffers;
- /** Base offset from the beginning of the indexable unit; for example, for ORC,
- * offset from the CB in a compressed file, from the stream in uncompressed file. */
- private int indexBaseOffset;
- /** Stream type; format-specific. */
- private int streamKind;
-
- /** Reference count. */
- private AtomicInteger refCount = new AtomicInteger(0);
-
- public void init(int kind) {
- streamKind = kind;
- indexBaseOffset = 0;
- }
-
- public void reset() {
- cacheBuffers.clear();
- refCount.set(0);
- }
-
- public void incRef() {
- refCount.incrementAndGet();
- }
-
- public int decRef() {
- int i = refCount.decrementAndGet();
- assert i >= 0;
- return i;
- }
-
- public List<MemoryBuffer> getCacheBuffers() {
- return cacheBuffers;
- }
-
- public void setCacheBuffers(List<MemoryBuffer> cacheBuffers) {
- this.cacheBuffers = cacheBuffers;
- }
-
- public int getIndexBaseOffset() {
- return indexBaseOffset;
- }
-
- public void setIndexBaseOffset(int indexBaseOffset) {
- this.indexBaseOffset = indexBaseOffset;
- }
-
- public int getStreamKind() {
- return streamKind;
- }
- }
-
- /** The key that is used to map this batch to source location. */
- protected BatchKey batchKey;
- /** Stream data for each stream, for each included column. */
- protected ColumnStreamData[][] columnData;
- /** Column indexes included in the batch. Correspond to columnData elements. */
- protected int[] columnIxs;
- // TODO: Maybe remove when solving the pooling issue.
- /** Generation version necessary to sync pooling reuse with the fact that two separate threads
- * operate on batches - the one that decodes them, and potential separate thread w/a "stop" call
- * that cleans them up. We don't want the decode thread to use the ECB that was thrown out and
- * reused, so it remembers the version and checks it after making sure no cleanup thread can ever
- * get to this ECB anymore. All this sync is ONLY needed because of high level cache code. */
- public int version = Integer.MIN_VALUE;
-
- public void reset() {
- if (columnData != null) {
- for (int i = 0; i < columnData.length; ++i) {
- columnData[i] = null;
- }
- }
- }
-
- public void initColumn(int colIxMod, int colIx, int streamCount) {
- columnIxs[colIxMod] = colIx;
- columnData[colIxMod] = new ColumnStreamData[streamCount];
- }
-
- public void setStreamData(int colIxMod, int streamIx, ColumnStreamData sb) {
- columnData[colIxMod][streamIx] = sb;
- }
-
- public void setAllStreamsData(int colIxMod, int colIx, ColumnStreamData[] sbs) {
- columnIxs[colIxMod] = colIx;
- columnData[colIxMod] = sbs;
- }
-
- public BatchKey getBatchKey() {
- return batchKey;
- }
-
- public ColumnStreamData[][] getColumnData() {
- return columnData;
- }
-
- public int[] getColumnIxs() {
- return columnIxs;
- }
-
- protected void resetColumnArrays(int columnCount) {
- if (columnIxs != null && columnCount == columnIxs.length) return;
- columnIxs = new int[columnCount];
- columnData = new ColumnStreamData[columnCount][];
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/common/src/java/org/apache/hadoop/hive/common/io/storage_api/MemoryBuffer.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/storage_api/MemoryBuffer.java b/common/src/java/org/apache/hadoop/hive/common/io/storage_api/MemoryBuffer.java
deleted file mode 100644
index 4dd2f09..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/io/storage_api/MemoryBuffer.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.common.io.storage_api;
-
-import java.nio.ByteBuffer;
-
-/** Abstract interface for any class wrapping a ByteBuffer. */
-public interface MemoryBuffer {
- /** Note - raw buffer should not be modified. */
- public ByteBuffer getByteBufferRaw();
- public ByteBuffer getByteBufferDup();
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
index 65854fc..08a6ca8 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
@@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/Cache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/Cache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/Cache.java
index 4d294b9..cee23a9 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/Cache.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/Cache.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.llap.cache;
-import org.apache.hadoop.hive.common.io.storage_api.EncodedColumnBatch.ColumnStreamData;
+import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData;
/** Dummy interface for now, might be different. */
public interface Cache<CacheKey> {
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionAwareAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionAwareAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionAwareAllocator.java
index 3baacfd..4e6e262 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionAwareAllocator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionAwareAllocator.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.hive.llap.cache;
-import org.apache.hadoop.hive.common.io.storage_api.Allocator;
-import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
+import org.apache.hadoop.hive.common.io.Allocator;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
/**
* An allocator that has additional, internal-only call to deallocate evicted buffer.
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapDataBuffer.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapDataBuffer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapDataBuffer.java
index 83eb0af..840aeab 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapDataBuffer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapDataBuffer.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.hive.llap.cache;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
import org.apache.hadoop.hive.llap.DebugUtils;
import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java
index 13944ff..b17edb5 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java
@@ -20,12 +20,12 @@ package org.apache.hadoop.hive.llap.cache;
import java.util.List;
-import org.apache.hadoop.hive.common.DiskRange;
-import org.apache.hadoop.hive.common.DiskRangeList;
-import org.apache.hadoop.hive.common.io.storage_api.DataCache.BooleanRef;
-import org.apache.hadoop.hive.common.io.storage_api.Allocator;
-import org.apache.hadoop.hive.common.io.storage_api.DataCache.DiskRangeListFactory;
-import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
+import org.apache.hadoop.hive.common.io.Allocator;
+import org.apache.hadoop.hive.common.io.DiskRange;
+import org.apache.hadoop.hive.common.io.DiskRangeList;
+import org.apache.hadoop.hive.common.io.DataCache.BooleanRef;
+import org.apache.hadoop.hive.common.io.DataCache.DiskRangeListFactory;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
public interface LowLevelCache {
public enum Priority {
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
index 6a54623..4d83bb9 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
@@ -25,13 +25,13 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hadoop.hive.common.DiskRange;
-import org.apache.hadoop.hive.common.DiskRangeList;
-import org.apache.hadoop.hive.common.DiskRangeList.MutateHelper;
-import org.apache.hadoop.hive.common.io.storage_api.Allocator;
-import org.apache.hadoop.hive.common.io.storage_api.DataCache.BooleanRef;
-import org.apache.hadoop.hive.common.io.storage_api.DataCache.DiskRangeListFactory;
-import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
+import org.apache.hadoop.hive.common.io.Allocator;
+import org.apache.hadoop.hive.common.io.DiskRange;
+import org.apache.hadoop.hive.common.io.DiskRangeList;
+import org.apache.hadoop.hive.common.io.DataCache.BooleanRef;
+import org.apache.hadoop.hive.common.io.DataCache.DiskRangeListFactory;
+import org.apache.hadoop.hive.common.io.DiskRangeList.MutateHelper;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
import org.apache.hadoop.hive.llap.DebugUtils;
import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/cache/NoopCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/NoopCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/NoopCache.java
index 6cd0c4a..d0461e8 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/NoopCache.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/NoopCache.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.llap.cache;
-import org.apache.hadoop.hive.common.io.storage_api.EncodedColumnBatch.ColumnStreamData;
+import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData;
public class NoopCache<CacheKey> implements Cache<CacheKey> {
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
index 333772c..322235f 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
-import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
+import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer;
import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.metadata.HiveException;
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
index 063eb08..d79b46a 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
@@ -26,7 +26,6 @@ import javax.management.ObjectName;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.io.storage_api.Allocator;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.llap.LogLevels;
import org.apache.hadoop.hive.llap.cache.BuddyAllocator;
@@ -47,7 +46,7 @@ import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics;
import org.apache.hadoop.hive.llap.metrics.MetricsUtils;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.io.orc.llap.OrcCacheKey;
+import org.apache.hadoop.hive.ql.io.orc.encoded.OrcCacheKey;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.metrics2.util.MBeans;
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ColumnVectorProducer.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ColumnVectorProducer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ColumnVectorProducer.java
index 79d3b32..5c34fa3 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ColumnVectorProducer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ColumnVectorProducer.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters;
import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch;
-import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
+import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.mapred.InputSplit;
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java
index b1d34ec..21a78fe 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java
@@ -22,11 +22,11 @@ import java.util.HashMap;
import java.util.List;
import java.util.concurrent.Callable;
-import org.apache.hadoop.hive.common.io.storage_api.EncodedColumnBatch;
+import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch;
import org.apache.hadoop.hive.llap.ConsumerFeedback;
import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics;
-import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
+import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer;
import org.apache.hive.common.util.FixedSizedObjectPool;
import org.apache.hive.common.util.FixedSizedObjectPool.PoolObjectHelper;
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java
index 765ade3..259c483 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java
@@ -32,8 +32,8 @@ import org.apache.hadoop.hive.llap.io.encoded.OrcEncodedDataReader;
import org.apache.hadoop.hive.llap.io.metadata.OrcMetadataCache;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics;
-import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
-import org.apache.hadoop.hive.ql.io.orc.llap.OrcCacheKey;
+import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer;
+import org.apache.hadoop.hive.ql.io.orc.encoded.OrcCacheKey;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.mapred.InputSplit;
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
index d51acb5..3408ac0 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
@@ -19,8 +19,8 @@ package org.apache.hadoop.hive.llap.io.decode;
import java.io.IOException;
-import org.apache.hadoop.hive.common.io.storage_api.EncodedColumnBatch;
-import org.apache.hadoop.hive.common.io.storage_api.EncodedColumnBatch.ColumnStreamData;
+import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch;
+import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData;
import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters;
import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch;
import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
@@ -29,9 +29,9 @@ import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.io.orc.CompressionCodec;
-import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl.OrcEncodedColumnBatch;
-import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
-import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
+import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer;
+import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey;
+import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedReaderImpl.OrcEncodedColumnBatch;
import org.apache.hadoop.hive.ql.io.orc.EncodedTreeReaderFactory;
import org.apache.hadoop.hive.ql.io.orc.OrcProto;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index f310fd5..8066dfd 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -13,13 +13,12 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.CallableWithNdc;
-import org.apache.hadoop.hive.common.DiskRange;
-import org.apache.hadoop.hive.common.DiskRangeList;
-import org.apache.hadoop.hive.common.io.storage_api.Allocator;
-import org.apache.hadoop.hive.common.io.storage_api.DataCache;
-import org.apache.hadoop.hive.common.io.storage_api.DataReader;
-import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
-import org.apache.hadoop.hive.common.io.storage_api.EncodedColumnBatch.ColumnStreamData;
+import org.apache.hadoop.hive.common.io.DataCache;
+import org.apache.hadoop.hive.common.io.Allocator;
+import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData;
+import org.apache.hadoop.hive.common.io.DiskRange;
+import org.apache.hadoop.hive.common.io.DiskRangeList;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.llap.ConsumerFeedback;
import org.apache.hadoop.hive.llap.DebugUtils;
@@ -37,9 +36,7 @@ import org.apache.hadoop.hive.ql.exec.DDLTask;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.HdfsUtils;
import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
-import org.apache.hadoop.hive.ql.io.orc.EncodedReader;
-import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl;
-import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl.OrcEncodedColumnBatch;
+import org.apache.hadoop.hive.ql.io.orc.DataReader;
import org.apache.hadoop.hive.ql.io.orc.MetadataReader;
import org.apache.hadoop.hive.ql.io.orc.OrcFile;
import org.apache.hadoop.hive.ql.io.orc.OrcFile.ReaderOptions;
@@ -47,17 +44,19 @@ import org.apache.hadoop.hive.ql.io.orc.OrcConf;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcProto;
import org.apache.hadoop.hive.ql.io.orc.OrcSplit;
-import org.apache.hadoop.hive.ql.io.orc.Reader;
+import org.apache.hadoop.hive.ql.io.orc.encoded.Reader;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.SargApplier;
-import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
-import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
-import org.apache.hadoop.hive.ql.io.orc.llap.OrcCacheKey;
+import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer;
+import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedOrcFile;
+import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedReader;
+import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedReaderImpl;
+import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey;
+import org.apache.hadoop.hive.ql.io.orc.encoded.OrcCacheKey;
+import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedReaderImpl.OrcEncodedColumnBatch;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderUtils;
import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-import org.apache.hadoop.hive.shims.HadoopShims;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
@@ -519,7 +518,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
}
long startTime = counters.startTimeCounter();
ReaderOptions opts = OrcFile.readerOptions(conf).filesystem(fs).fileMetadata(fileMetadata);
- orcReader = OrcFile.createReader(path, opts);
+ orcReader = EncodedOrcFile.createReader(path, opts);
counters.incrTimeCounter(Counter.HDFS_TIME_US, startTime);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
index 635ba6c..43c8fb3 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
@@ -24,7 +24,7 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.hive.llap.cache.LowLevelCachePolicy;
import org.apache.hadoop.hive.llap.cache.MemoryManager;
import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
-import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
+import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey;
public class OrcMetadataCache {
private final ConcurrentHashMap<Long, OrcFileMetadata> metadata =
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
index 762c9f9..d70ff1a 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream;
import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
-import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
+import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey;
import com.google.common.annotations.VisibleForTesting;
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
index 3299135..d4d4bb2 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
@@ -30,8 +30,8 @@ import java.util.concurrent.FutureTask;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
-import org.apache.hadoop.hive.common.io.storage_api.Allocator.AllocatorOutOfMemoryException;
+import org.apache.hadoop.hive.common.io.Allocator.AllocatorOutOfMemoryException;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
index 382e948..ef51869 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.hive.ql.io.orc.OrcProto.TimestampStatistics;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.Index;
import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
-import org.apache.hadoop.hive.ql.io.orc.llap.OrcBatchKey;
+import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.junit.Test;
import org.mockito.Mockito;
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
index d4b8a46..94684ca 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
@@ -33,14 +33,14 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.common.DiskRange;
-import org.apache.hadoop.hive.common.DiskRangeList;
-import org.apache.hadoop.hive.common.DiskRangeList.CreateHelper;
-import org.apache.hadoop.hive.common.io.storage_api.DataCache.DiskRangeListFactory;
-import org.apache.hadoop.hive.common.io.storage_api.MemoryBuffer;
+import org.apache.hadoop.hive.common.io.DiskRange;
+import org.apache.hadoop.hive.common.io.DiskRangeList;
+import org.apache.hadoop.hive.common.io.DataCache.DiskRangeListFactory;
+import org.apache.hadoop.hive.common.io.DiskRangeList.CreateHelper;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
-import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl.CacheChunk;
+import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedReaderImpl.CacheChunk;
import org.junit.Test;
public class TestLowLevelCacheImpl {
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java b/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
index 8dc62d8..a5291ee 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/DebugUtils.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.llap;
-import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl;
+import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedReaderImpl;
/**
* A class that contains debug methods; also allows enabling the logging of various
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DataReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DataReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DataReader.java
new file mode 100644
index 0000000..e0d9943
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DataReader.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.orc;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hive.common.io.DiskRangeList;
+
+/** An abstract data reader that IO formats can use to read bytes from underlying storage. */
+public interface DataReader {
+
+ /** Opens the DataReader, making it ready to use. */
+ void open() throws IOException;
+
+ /** Closes the DataReader. */
+ void close() throws IOException;
+
+ /** Reads the data.
+ *
+ * Note that for the cases such as zero-copy read, caller must release the disk ranges
+ * produced after being done with them. Call isTrackingDiskRanges to find out if this is needed.
+ * @param range List if disk ranges to read. Ranges with data will be ignored.
+ * @param baseOffset Base offset from the start of the file of the ranges in disk range list.
+ * @param doForceDirect Whether the data should be read into direct buffers.
+ * @return New or modified list of DiskRange-s, where all the ranges are filled with data.
+ */
+ DiskRangeList readFileData(
+ DiskRangeList range, long baseOffset, boolean doForceDirect) throws IOException;
+
+
+ /**
+ * Whether the user should release buffers created by readFileData. See readFileData javadoc.
+ */
+ boolean isTrackingDiskRanges();
+
+ /**
+ * Releases buffers created by readFileData. See readFileData javadoc.
+ * @param toRelease The buffer to release.
+ */
+ void releaseBuffer(ByteBuffer toRelease);
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/2374cfb7/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java
deleted file mode 100644
index a5cead4..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReader.java
+++ /dev/null
@@ -1,22 +0,0 @@
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hive.ql.io.orc.EncodedReaderImpl.OrcEncodedColumnBatch;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream;
-import org.apache.hadoop.hive.ql.io.orc.llap.Consumer;
-
-public interface EncodedReader {
- // TODO: document
- void readEncodedColumns(int stripeIx, StripeInformation stripe,
- RowIndex[] index, List<ColumnEncoding> encodings, List<Stream> streams,
- boolean[] included, boolean[][] colRgs,
- Consumer<OrcEncodedColumnBatch> consumer) throws IOException;
-
- void close() throws IOException;
-
- void setDebugTracing(boolean isEnabled);
-}
\ No newline at end of file