You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2016/10/20 13:51:18 UTC
[1/3] hadoop git commit: HADOOP-13735
ITestS3AFileContextStatistics.testStatistics() failing. Contributed by Pieter
Reuse
Repository: hadoop
Updated Branches:
refs/heads/branch-2 5a37668d1 -> c640ba5fc
refs/heads/branch-2.8 f5b080e15 -> 769babf47
refs/heads/trunk 73504b1bd -> 9ae270af0
HADOOP-13735 ITestS3AFileContextStatistics.testStatistics() failing. Contributed by Pieter Reuse
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c640ba5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c640ba5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c640ba5f
Branch: refs/heads/branch-2
Commit: c640ba5fc954e7f6c23ce4a8b62a6c11f5f14df9
Parents: 5a37668
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 20 14:50:30 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 20 14:50:30 2016 +0100
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +-
.../org/apache/hadoop/fs/s3a/S3AInstrumentation.java | 13 ++++++++++---
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c640ba5f/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 1532cde..0e899e3e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -547,7 +547,7 @@ public class S3AFileSystem extends FileSystem {
progress,
partSize,
blockFactory,
- instrumentation.newOutputStreamStatistics(),
+ instrumentation.newOutputStreamStatistics(statistics),
new WriteOperationHelper(key)
),
null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c640ba5f/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
index 963c53f..fb8c852 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
@@ -37,6 +37,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.fs.FileSystem.Statistics;
import static org.apache.hadoop.fs.s3a.Statistic.*;
@@ -639,9 +640,8 @@ public class S3AInstrumentation {
* Create a stream output statistics instance.
* @return the new instance
*/
-
- OutputStreamStatistics newOutputStreamStatistics() {
- return new OutputStreamStatistics();
+ OutputStreamStatistics newOutputStreamStatistics(Statistics statistics) {
+ return new OutputStreamStatistics(statistics);
}
/**
@@ -677,6 +677,12 @@ public class S3AInstrumentation {
private final AtomicLong queueDuration = new AtomicLong(0);
private final AtomicLong exceptionsInMultipartFinalize = new AtomicLong(0);
+ private Statistics statistics;
+
+ public OutputStreamStatistics(Statistics statistics){
+ this.statistics = statistics;
+ }
+
/**
* Block is queued for upload.
*/
@@ -717,6 +723,7 @@ public class S3AInstrumentation {
/** Intermediate report of bytes uploaded. */
void bytesTransferred(long byteCount) {
bytesUploaded.addAndGet(byteCount);
+ statistics.incrementBytesWritten(byteCount);
bytesPendingUpload.addAndGet(-byteCount);
incrementGauge(STREAM_WRITE_BLOCK_UPLOADS_DATA_PENDING, -byteCount);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/3] hadoop git commit: HADOOP-13735
ITestS3AFileContextStatistics.testStatistics() failing. Contributed by Pieter
Reuse
Posted by st...@apache.org.
HADOOP-13735 ITestS3AFileContextStatistics.testStatistics() failing. Contributed by Pieter Reuse
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ae270af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ae270af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ae270af
Branch: refs/heads/trunk
Commit: 9ae270af02c243993f853513c731cb268430e492
Parents: 73504b1
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 20 14:50:30 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 20 14:50:59 2016 +0100
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +-
.../org/apache/hadoop/fs/s3a/S3AInstrumentation.java | 13 ++++++++++---
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae270af/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 2354819..9908ba7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -548,7 +548,7 @@ public class S3AFileSystem extends FileSystem {
progress,
partSize,
blockFactory,
- instrumentation.newOutputStreamStatistics(),
+ instrumentation.newOutputStreamStatistics(statistics),
new WriteOperationHelper(key)
),
null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ae270af/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
index 963c53f..fb8c852 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
@@ -37,6 +37,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.fs.FileSystem.Statistics;
import static org.apache.hadoop.fs.s3a.Statistic.*;
@@ -639,9 +640,8 @@ public class S3AInstrumentation {
* Create a stream output statistics instance.
* @return the new instance
*/
-
- OutputStreamStatistics newOutputStreamStatistics() {
- return new OutputStreamStatistics();
+ OutputStreamStatistics newOutputStreamStatistics(Statistics statistics) {
+ return new OutputStreamStatistics(statistics);
}
/**
@@ -677,6 +677,12 @@ public class S3AInstrumentation {
private final AtomicLong queueDuration = new AtomicLong(0);
private final AtomicLong exceptionsInMultipartFinalize = new AtomicLong(0);
+ private Statistics statistics;
+
+ public OutputStreamStatistics(Statistics statistics){
+ this.statistics = statistics;
+ }
+
/**
* Block is queued for upload.
*/
@@ -717,6 +723,7 @@ public class S3AInstrumentation {
/** Intermediate report of bytes uploaded. */
void bytesTransferred(long byteCount) {
bytesUploaded.addAndGet(byteCount);
+ statistics.incrementBytesWritten(byteCount);
bytesPendingUpload.addAndGet(-byteCount);
incrementGauge(STREAM_WRITE_BLOCK_UPLOADS_DATA_PENDING, -byteCount);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/3] hadoop git commit: HADOOP-13735
ITestS3AFileContextStatistics.testStatistics() failing. Contributed by Pieter
Reuse
Posted by st...@apache.org.
HADOOP-13735 ITestS3AFileContextStatistics.testStatistics() failing. Contributed by Pieter Reuse
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/769babf4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/769babf4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/769babf4
Branch: refs/heads/branch-2.8
Commit: 769babf47063c3f922c09f59ae9b9b94567c94f7
Parents: f5b080e
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 20 14:50:30 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 20 14:50:44 2016 +0100
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +-
.../org/apache/hadoop/fs/s3a/S3AInstrumentation.java | 13 ++++++++++---
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/769babf4/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 1532cde..0e899e3e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -547,7 +547,7 @@ public class S3AFileSystem extends FileSystem {
progress,
partSize,
blockFactory,
- instrumentation.newOutputStreamStatistics(),
+ instrumentation.newOutputStreamStatistics(statistics),
new WriteOperationHelper(key)
),
null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/769babf4/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
index 963c53f..fb8c852 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
@@ -37,6 +37,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.fs.FileSystem.Statistics;
import static org.apache.hadoop.fs.s3a.Statistic.*;
@@ -639,9 +640,8 @@ public class S3AInstrumentation {
* Create a stream output statistics instance.
* @return the new instance
*/
-
- OutputStreamStatistics newOutputStreamStatistics() {
- return new OutputStreamStatistics();
+ OutputStreamStatistics newOutputStreamStatistics(Statistics statistics) {
+ return new OutputStreamStatistics(statistics);
}
/**
@@ -677,6 +677,12 @@ public class S3AInstrumentation {
private final AtomicLong queueDuration = new AtomicLong(0);
private final AtomicLong exceptionsInMultipartFinalize = new AtomicLong(0);
+ private Statistics statistics;
+
+ public OutputStreamStatistics(Statistics statistics){
+ this.statistics = statistics;
+ }
+
/**
* Block is queued for upload.
*/
@@ -717,6 +723,7 @@ public class S3AInstrumentation {
/** Intermediate report of bytes uploaded. */
void bytesTransferred(long byteCount) {
bytesUploaded.addAndGet(byteCount);
+ statistics.incrementBytesWritten(byteCount);
bytesPendingUpload.addAndGet(-byteCount);
incrementGauge(STREAM_WRITE_BLOCK_UPLOADS_DATA_PENDING, -byteCount);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org