You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2019/05/09 20:16:52 UTC
[hadoop] branch branch-3.1 updated: HADOOP-16278. With S3A
Filesystem, Long Running services End up Doing lot of GC and eventually die.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 8e0fdae HADOOP-16278. With S3A Filesystem, Long Running services End up Doing lot of GC and eventually die.
8e0fdae is described below
commit 8e0fdaeb389a93df02215cf16fffe4b777621028
Author: Rajat Khandelwal <pr...@apache.org>
AuthorDate: Thu May 9 21:15:09 2019 +0100
HADOOP-16278. With S3A Filesystem, Long Running services End up Doing lot of GC and eventually die.
Contributed by Rajat Khandelwal
(cherry picked from commit 591ca698230f25217c10c7549aff8097baa11f1e)
---
.../main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
index 26ecefd..97787ab 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
@@ -120,7 +120,8 @@ public class S3AInstrumentation implements Closeable, MetricsSource {
private final MutableCounterLong streamBytesReadInClose;
private final MutableCounterLong streamBytesDiscardedInAbort;
private final MutableCounterLong ignoredErrors;
-
+ private final MutableQuantiles putLatencyQuantile;
+ private final MutableQuantiles throttleRateQuantile;
private final MutableCounterLong numberOfFilesCreated;
private final MutableCounterLong numberOfFilesCopied;
private final MutableCounterLong bytesOfFilesCopied;
@@ -235,9 +236,9 @@ public class S3AInstrumentation implements Closeable, MetricsSource {
}
//todo need a config for the quantiles interval?
int interval = 1;
- quantiles(S3GUARD_METADATASTORE_PUT_PATH_LATENCY,
+ putLatencyQuantile = quantiles(S3GUARD_METADATASTORE_PUT_PATH_LATENCY,
"ops", "latency", interval);
- quantiles(S3GUARD_METADATASTORE_THROTTLE_RATE,
+ throttleRateQuantile = quantiles(S3GUARD_METADATASTORE_THROTTLE_RATE,
"events", "frequency (Hz)", interval);
registerAsMetricsSource(name);
@@ -600,6 +601,8 @@ public class S3AInstrumentation implements Closeable, MetricsSource {
public void close() {
synchronized (metricsSystemLock) {
+ putLatencyQuantile.stop();
+ throttleRateQuantile.stop();
metricsSystem.unregisterSource(metricsSourceName);
int activeSources = --metricsSourceActiveCounter;
if (activeSources == 0) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org