You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2015/04/01 23:10:18 UTC
hadoop git commit: HDFS-8026. Trace
FSOutputSummer#writeChecksumChunks rather than DFSOutputStream#writeChunk
(cmccabe)
Repository: hadoop
Updated Branches:
refs/heads/trunk 8366a36ad -> c94d594a5
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than DFSOutputStream#writeChunk (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c94d594a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c94d594a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c94d594a
Branch: refs/heads/trunk
Commit: c94d594a57806dec515e2a2053a1221f8ce48cc4
Parents: 8366a36
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Apr 1 13:55:40 2015 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Wed Apr 1 14:10:10 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/FSOutputSummer.java | 20 ++++++++++++++++----
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 15 ++++-----------
.../org/apache/hadoop/tracing/TestTracing.java | 4 ++--
4 files changed, 25 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c94d594a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
index 13a5e26..d2998b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.fs;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.DataChecksum;
+import org.apache.htrace.NullScope;
+import org.apache.htrace.TraceScope;
import java.io.IOException;
import java.io.OutputStream;
@@ -194,16 +196,26 @@ abstract public class FSOutputSummer extends OutputStream {
return sum.getChecksumSize();
}
+ protected TraceScope createWriteTraceScope() {
+ return NullScope.INSTANCE;
+ }
+
/** Generate checksums for the given data chunks and output chunks & checksums
* to the underlying output stream.
*/
private void writeChecksumChunks(byte b[], int off, int len)
throws IOException {
sum.calculateChunkedSums(b, off, len, checksum, 0);
- for (int i = 0; i < len; i += sum.getBytesPerChecksum()) {
- int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i);
- int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize();
- writeChunk(b, off + i, chunkLen, checksum, ckOffset, getChecksumSize());
+ TraceScope scope = createWriteTraceScope();
+ try {
+ for (int i = 0; i < len; i += sum.getBytesPerChecksum()) {
+ int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i);
+ int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize();
+ writeChunk(b, off + i, chunkLen, checksum, ckOffset,
+ getChecksumSize());
+ }
+ } finally {
+ scope.close();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c94d594a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 435fdd7..b5591e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -378,6 +378,9 @@ Release 2.8.0 - UNRELEASED
OPTIMIZATIONS
+ HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
+ DFSOutputStream#writeChunk (cmccabe)
+
BUG FIXES
HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c94d594a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 933d8e6..c88639d 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -372,21 +372,14 @@ public class DFSOutputStream extends FSOutputSummer
}
}
+ protected TraceScope createWriteTraceScope() {
+ return dfsClient.getPathTraceScope("DFSOutputStream#write", src);
+ }
+
// @see FSOutputSummer#writeChunk()
@Override
protected synchronized void writeChunk(byte[] b, int offset, int len,
byte[] checksum, int ckoff, int cklen) throws IOException {
- TraceScope scope =
- dfsClient.getPathTraceScope("DFSOutputStream#writeChunk", src);
- try {
- writeChunkImpl(b, offset, len, checksum, ckoff, cklen);
- } finally {
- scope.close();
- }
- }
-
- private synchronized void writeChunkImpl(byte[] b, int offset, int len,
- byte[] checksum, int ckoff, int cklen) throws IOException {
dfsClient.checkOpen();
checkClosed();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c94d594a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
index 3720abe..01361b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
@@ -89,7 +89,7 @@ public class TestTracing {
"org.apache.hadoop.hdfs.protocol.ClientProtocol.complete",
"ClientNamenodeProtocol#complete",
"newStreamForCreate",
- "DFSOutputStream#writeChunk",
+ "DFSOutputStream#write",
"DFSOutputStream#close",
"dataStreamer",
"OpWriteBlockProto",
@@ -117,7 +117,7 @@ public class TestTracing {
"org.apache.hadoop.hdfs.protocol.ClientProtocol.complete",
"ClientNamenodeProtocol#complete",
"newStreamForCreate",
- "DFSOutputStream#writeChunk",
+ "DFSOutputStream#write",
"DFSOutputStream#close",
};
for (String desc : spansInTopTrace) {