You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by jl...@apache.org on 2018/03/26 19:51:48 UTC
tez git commit: TEZ-3907. Improve log message to include the location
the writers decide to spill output (Kuhu Shukla via jlowe)
Repository: tez
Updated Branches:
refs/heads/master 85bd772c4 -> 7f936f614
TEZ-3907. Improve log message to include the location the writers decide to spill output (Kuhu Shukla via jlowe)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/7f936f61
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/7f936f61
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/7f936f61
Branch: refs/heads/master
Commit: 7f936f614914a456e3ace9ca5a5f0d218e7c7e01
Parents: 85bd772
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Mar 26 14:49:12 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon Mar 26 14:49:12 2018 -0500
----------------------------------------------------------------------
.../tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java | 3 ++-
.../library/common/writers/UnorderedPartitionedKVWriter.java | 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/7f936f61/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
index cfcbd56..557a538 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
@@ -977,7 +977,8 @@ public final class DefaultSorter extends ExternalSorter implements IndexedSortab
totalIndexCacheMemory +=
spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH;
}
- LOG.info(outputContext.getDestinationVertexName() + ": " + "Finished spill " + numSpills);
+ LOG.info(outputContext.getDestinationVertexName() + ": " + "Finished spill " + numSpills
+ + " at " + filename.toString());
++numSpills;
if (!isFinalMergeEnabled()) {
numShuffleChunks.setValue(numSpills);
http://git-wip-us.apache.org/repos/asf/tez/blob/7f936f61/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
index b9f0edf..948417d 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
@@ -590,6 +590,7 @@ public class UnorderedPartitionedKVWriter extends BaseUnorderedPartitionedKVWrit
this.spillPathDetails = getSpillPathDetails(false, -1, spillNumber);
this.spillIndex = spillPathDetails.spillIndex;
}
+ LOG.info("Writing spill " + spillNumber + " to " + spillPathDetails.outputFilePath.toString());
FSDataOutputStream out = rfs.create(spillPathDetails.outputFilePath);
if (!SPILL_FILE_PERMS.equals(SPILL_FILE_PERMS.applyUMask(FsPermission.getUMask(conf)))) {
rfs.setPermission(spillPathDetails.outputFilePath, SPILL_FILE_PERMS);