You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by rb...@apache.org on 2015/10/22 23:13:36 UTC
tez git commit: TEZ-2907. NPE in IFile.Reader.getLength during final
merge operation (rbalamohan)
Repository: tez
Updated Branches:
refs/heads/master 01ad29e44 -> f3cea630e
TEZ-2907. NPE in IFile.Reader.getLength during final merge operation (rbalamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/f3cea630
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/f3cea630
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/f3cea630
Branch: refs/heads/master
Commit: f3cea630e8329018a1e600859353576cb83cf77d
Parents: 01ad29e
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Thu Oct 22 13:59:05 2015 -0700
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Thu Oct 22 13:59:05 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +++
.../library/common/shuffle/orderedgrouped/MergeManager.java | 6 ++++++
2 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/f3cea630/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index aec51ed..d19df34 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ Release 0.8.2: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2907. NPE in IFile.Reader.getLength during final merge operation
TEZ-2903. Stop using proprietary APIs in RPCLoadGen.
TEZ-2882. Consider improving fetch failure handling
TEZ-2850. Tez MergeManager OOM for small Map Outputs
@@ -220,6 +221,7 @@ Release 0.7.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES
+ TEZ-2907. NPE in IFile.Reader.getLength during final merge operation
TEZ-2850. Tez MergeManager OOM for small Map Outputs
TEZ-2886. Ability to merge AM credentials with DAG credentials.
TEZ-2896. Fix thread names used during Input/Output initialization.
@@ -504,6 +506,7 @@ Release 0.6.3: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2907. NPE in IFile.Reader.getLength during final merge operation
TEZ-2850. Tez MergeManager OOM for small Map Outputs
TEZ-2781. Fallback to send only TaskAttemptFailedEvent if taskFailed heartbeat fails
TEZ-2855. Fix a potential NPE while routing VertexManager events.
http://git-wip-us.apache.org/repos/asf/tez/blob/f3cea630/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
index eb2cece..fb9b243 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
@@ -949,12 +949,14 @@ public class MergeManager implements FetchedInputAllocatorOrderedGrouped {
class RawKVIteratorReader extends IFile.Reader {
private final TezRawKeyValueIterator kvIter;
+ private final long size;
public RawKVIteratorReader(TezRawKeyValueIterator kvIter, long size)
throws IOException {
super(null, size, null, spilledRecordsCounter, null, ifileReadAhead,
ifileReadAheadLength, ifileBufferSize);
this.kvIter = kvIter;
+ this.size = size;
}
@Override
public KeyState readRawKey(DataInputBuffer key) throws IOException {
@@ -982,6 +984,10 @@ public class MergeManager implements FetchedInputAllocatorOrderedGrouped {
public void close() throws IOException {
kvIter.close();
}
+
+ @Override public long getLength() {
+ return size;
+ }
}
private TezRawKeyValueIterator finalMerge(Configuration job, FileSystem fs,