You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/09/29 17:51:31 UTC
hive git commit: HIVE-11945: ORC with non-local reads may not be
reusing connection to DN (Rajesh Balamohan reviewed by Sergey Shelukhin and
Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/branch-1 67e33f3e9 -> 1d89323df
HIVE-11945: ORC with non-local reads may not be reusing connection to DN (Rajesh Balamohan reviewed by Sergey Shelukhin and Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1d89323d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1d89323d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1d89323d
Branch: refs/heads/branch-1
Commit: 1d89323dfd8551b957607eeb7a4ea1757c1d1d5f
Parents: 67e33f3
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Tue Sep 29 10:51:12 2015 -0500
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Tue Sep 29 10:51:12 2015 -0500
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/io/orc/MetadataReader.java | 6 ++----
.../org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java | 13 +++++--------
.../hadoop/hive/ql/io/orc/RecordReaderUtils.java | 5 +++--
3 files changed, 10 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1d89323d/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
index cdc0372..43d2933 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
@@ -82,8 +82,7 @@ public class MetadataReader {
}
if ((included == null || included[col]) && indexes[col] == null) {
byte[] buffer = new byte[len];
- file.seek(offset);
- file.readFully(buffer);
+ file.readFully(offset, buffer, 0, buffer.length);
ByteBuffer[] bb = new ByteBuffer[] {ByteBuffer.wrap(buffer)};
indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create("index",
bb, new long[]{0}, stream.getLength(), codec, bufferSize));
@@ -108,8 +107,7 @@ public class MetadataReader {
// read the footer
ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
- file.seek(offset);
- file.readFully(tailBuf.array(), tailBuf.arrayOffset(), tailLength);
+ file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
return OrcProto.StripeFooter.parseFrom(InStream.create("footer",
Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)),
tailLength, codec, bufferSize));
http://git-wip-us.apache.org/repos/asf/hive/blob/1d89323d/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
index c990d85..6589e41 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
@@ -243,9 +243,8 @@ public class ReaderImpl implements Reader {
if (!Text.decode(array, offset, len).equals(OrcFile.MAGIC)) {
// If it isn't there, this may be the 0.11.0 version of ORC.
// Read the first 3 bytes of the file to check for the header
- in.seek(0);
byte[] header = new byte[len];
- in.readFully(header, 0, len);
+ in.readFully(0, header, 0, len);
// if it isn't there, this isn't an ORC file
if (!Text.decode(header, 0 , len).equals(OrcFile.MAGIC)) {
throw new FileFormatException("Malformed ORC file " + path +
@@ -365,10 +364,9 @@ public class ReaderImpl implements Reader {
//read last bytes into buffer to get PostScript
int readSize = (int) Math.min(size, DIRECTORY_SIZE_GUESS);
- file.seek(size - readSize);
ByteBuffer buffer = ByteBuffer.allocate(readSize);
- file.readFully(buffer.array(), buffer.arrayOffset() + buffer.position(),
- buffer.remaining());
+ file.readFully((size - readSize),
+ buffer.array(), buffer.arrayOffset(), readSize);
//read the PostScript
//get length of PostScript
@@ -408,10 +406,9 @@ public class ReaderImpl implements Reader {
int extra = Math.max(0, psLen + 1 + footerSize + metadataSize - readSize);
if (extra > 0) {
//more bytes need to be read, seek back to the right place and read extra bytes
- file.seek(size - readSize - extra);
ByteBuffer extraBuf = ByteBuffer.allocate(extra + readSize);
- file.readFully(extraBuf.array(),
- extraBuf.arrayOffset() + extraBuf.position(), extra);
+ file.readFully((size - readSize - extra), extraBuf.array(),
+ extraBuf.arrayOffset() + extraBuf.position(), extra);
extraBuf.position(extra);
//append with already read bytes
extraBuf.put(buffer);
http://git-wip-us.apache.org/repos/asf/hive/blob/1d89323d/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
index ded3979..9c9a1c0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
@@ -246,8 +246,8 @@ public class RecordReaderUtils {
}
int len = (int) (range.getEnd() - range.getOffset());
long off = range.getOffset();
- file.seek(base + off);
if (zcr != null) {
+ file.seek(base + off);
boolean hasReplaced = false;
while (len > 0) {
ByteBuffer partial = zcr.readBuffer(len, false);
@@ -264,12 +264,13 @@ public class RecordReaderUtils {
off += read;
}
} else if (doForceDirect) {
+ file.seek(base + off);
ByteBuffer directBuf = ByteBuffer.allocateDirect(len);
readDirect(file, len, directBuf);
range = range.replaceSelfWith(new BufferChunk(directBuf, range.getOffset()));
} else {
byte[] buffer = new byte[len];
- file.readFully(buffer, 0, buffer.length);
+ file.readFully((base + off), buffer, 0, buffer.length);
range = range.replaceSelfWith(new BufferChunk(ByteBuffer.wrap(buffer), range.getOffset()));
}
range = range.next;