You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by rb...@apache.org on 2015/06/02 10:19:10 UTC
tez git commit: TEZ-2505. PipelinedSorter uses Comparator objects
concurrently from multiple threads (rbalamohan)
Repository: tez
Updated Branches:
refs/heads/master 4c48283ee -> 237d1895d
TEZ-2505. PipelinedSorter uses Comparator objects concurrently from multiple threads (rbalamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/237d1895
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/237d1895
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/237d1895
Branch: refs/heads/master
Commit: 237d1895d224fc07466e0c4e494960b32d7a69aa
Parents: 4c48283
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Tue Jun 2 13:50:16 2015 +0530
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Tue Jun 2 13:50:16 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../runtime/library/common/sort/impl/PipelinedSorter.java | 10 ++++------
2 files changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/237d1895/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 116a387..60e2b89 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -31,6 +31,7 @@ Release 0.7.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2505. PipelinedSorter uses Comparator objects concurrently from multiple threads.
TEZ-2504. Tez UI: tables - show status column without scrolling, numeric 0 shown as Not available
TEZ-2478. Move OneToOne routing to store events in Tasks.
TEZ-2482. Tez UI: Mouse events not working on IE11
http://git-wip-us.apache.org/repos/asf/tez/blob/237d1895/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
index 6e4d72e..e3824be 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
@@ -107,8 +107,6 @@ public class PipelinedSorter extends ExternalSorter {
private final ArrayList<TezSpillRecord> indexCacheList =
new ArrayList<TezSpillRecord>();
- private int totalIndexCacheMemory;
- private int indexCacheMemoryLimit;
private final boolean pipelinedShuffle;
@@ -133,8 +131,6 @@ public class PipelinedSorter extends ExternalSorter {
//sanity checks
final long sortmb = this.availableMemoryMb;
- indexCacheMemoryLimit = this.conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES,
- TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES_DEFAULT);
// buffers and accounting
long maxMemUsage = sortmb << 20;
@@ -244,7 +240,8 @@ public class PipelinedSorter extends ExternalSorter {
}
}
Preconditions.checkArgument(listIterator.hasNext(), "block iterator should not be empty");
- span = new SortSpan((ByteBuffer)listIterator.next().clear(), (1024*1024), perItem, this.comparator);
+ span = new SortSpan((ByteBuffer)listIterator.next().clear(), (1024*1024),
+ perItem, ConfigUtils.getIntermediateOutputKeyComparator(this.conf));
} else {
// queue up the sort
SortTask task = new SortTask(span, sorter);
@@ -755,7 +752,8 @@ public class PipelinedSorter extends ExternalSorter {
items = 1024*1024;
perItem = 16;
}
- newSpan = new SortSpan(remaining, items, perItem, this.comparator);
+ newSpan = new SortSpan(remaining, items, perItem,
+ ConfigUtils.getIntermediateOutputKeyComparator(conf));
newSpan.index = index+1;
LOG.info(String.format("New Span%d.length = %d, perItem = %d", newSpan.index, newSpan
.length(), perItem) + ", counter:" + mapOutputRecordCounter.getValue());