You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/09/29 03:11:03 UTC
hive git commit: HIVE-11945: ORC with non-local reads may not be
reusing connection to DN (Rajesh Balamohan reviewed by Sergey Shelukhin,
Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master 392145812 -> a4c43f033
HIVE-11945: ORC with non-local reads may not be reusing connection to DN (Rajesh Balamohan reviewed by Sergey Shelukhin, Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a4c43f03
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a4c43f03
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a4c43f03
Branch: refs/heads/master
Commit: a4c43f0335b33a75d2e9f3dc53b3cd33f8f115cf
Parents: 3921458
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Mon Sep 28 20:10:50 2015 -0500
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Mon Sep 28 20:10:50 2015 -0500
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/io/orc/MetadataReader.java | 6 ++----
.../org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java | 12 +++++-------
.../apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java | 5 +++--
3 files changed, 10 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a4c43f03/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
index cdc0372..43d2933 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
@@ -82,8 +82,7 @@ public class MetadataReader {
}
if ((included == null || included[col]) && indexes[col] == null) {
byte[] buffer = new byte[len];
- file.seek(offset);
- file.readFully(buffer);
+ file.readFully(offset, buffer, 0, buffer.length);
ByteBuffer[] bb = new ByteBuffer[] {ByteBuffer.wrap(buffer)};
indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create("index",
bb, new long[]{0}, stream.getLength(), codec, bufferSize));
@@ -108,8 +107,7 @@ public class MetadataReader {
// read the footer
ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
- file.seek(offset);
- file.readFully(tailBuf.array(), tailBuf.arrayOffset(), tailLength);
+ file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
return OrcProto.StripeFooter.parseFrom(InStream.create("footer",
Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)),
tailLength, codec, bufferSize));
http://git-wip-us.apache.org/repos/asf/hive/blob/a4c43f03/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
index ab539c4..23b3b55 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
@@ -244,9 +244,8 @@ public class ReaderImpl implements Reader {
if (!Text.decode(array, offset, len).equals(OrcFile.MAGIC)) {
// If it isn't there, this may be the 0.11.0 version of ORC.
// Read the first 3 bytes of the file to check for the header
- in.seek(0);
byte[] header = new byte[len];
- in.readFully(header, 0, len);
+ in.readFully(0, header, 0, len);
// if it isn't there, this isn't an ORC file
if (!Text.decode(header, 0 , len).equals(OrcFile.MAGIC)) {
throw new FileFormatException("Malformed ORC file " + path +
@@ -472,10 +471,10 @@ public class ReaderImpl implements Reader {
//read last bytes into buffer to get PostScript
int readSize = (int) Math.min(size, DIRECTORY_SIZE_GUESS);
- file.seek(size - readSize);
ByteBuffer buffer = ByteBuffer.allocate(readSize);
assert buffer.position() == 0;
- file.readFully(buffer.array(), buffer.arrayOffset(), readSize);
+ file.readFully((size - readSize),
+ buffer.array(), buffer.arrayOffset(), readSize);
buffer.position(0);
//read the PostScript
@@ -495,10 +494,9 @@ public class ReaderImpl implements Reader {
int extra = Math.max(0, psLen + 1 + footerSize + metadataSize - readSize);
if (extra > 0) {
//more bytes need to be read, seek back to the right place and read extra bytes
- file.seek(size - readSize - extra);
ByteBuffer extraBuf = ByteBuffer.allocate(extra + readSize);
- file.readFully(extraBuf.array(),
- extraBuf.arrayOffset() + extraBuf.position(), extra);
+ file.readFully((size - readSize - extra), extraBuf.array(),
+ extraBuf.arrayOffset() + extraBuf.position(), extra);
extraBuf.position(extra);
//append with already read bytes
extraBuf.put(buffer);
http://git-wip-us.apache.org/repos/asf/hive/blob/a4c43f03/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
index ded3979..9c9a1c0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
@@ -246,8 +246,8 @@ public class RecordReaderUtils {
}
int len = (int) (range.getEnd() - range.getOffset());
long off = range.getOffset();
- file.seek(base + off);
if (zcr != null) {
+ file.seek(base + off);
boolean hasReplaced = false;
while (len > 0) {
ByteBuffer partial = zcr.readBuffer(len, false);
@@ -264,12 +264,13 @@ public class RecordReaderUtils {
off += read;
}
} else if (doForceDirect) {
+ file.seek(base + off);
ByteBuffer directBuf = ByteBuffer.allocateDirect(len);
readDirect(file, len, directBuf);
range = range.replaceSelfWith(new BufferChunk(directBuf, range.getOffset()));
} else {
byte[] buffer = new byte[len];
- file.readFully(buffer, 0, buffer.length);
+ file.readFully((base + off), buffer, 0, buffer.length);
range = range.replaceSelfWith(new BufferChunk(ByteBuffer.wrap(buffer), range.getOffset()));
}
range = range.next;