You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2017/05/11 02:17:23 UTC
hadoop git commit: HADOOP-12975. Add jitter to CachingGetSpaceUsed's
thread (Elliott Clark via Colin P. McCabe). HADOOP-14408. Backport
contributed by Erik Krogen.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 f052976ff -> 834d4217b
HADOOP-12975. Add jitter to CachingGetSpaceUsed's thread (Elliott Clark via Colin P. McCabe).
HADOOP-14408. Backport contributed by Erik Krogen.
(cherry picked from commit bf780406f2b30e627bdf36ac07973f6931f81106)
(cherry picked from commit 56c997a1a61188693ed306b3ceddb13a2f2e6456)
(cherry picked from commit 04682cc6b00e2ba9c0a0a3d1f71a06f36cd4c1a9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/834d4217
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/834d4217
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/834d4217
Branch: refs/heads/branch-2.7
Commit: 834d4217b75463648452532990c6f910d77c8180
Parents: f052976
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Wed May 10 18:20:26 2017 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Wed May 10 19:16:01 2017 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 4 +++
.../src/main/java/org/apache/hadoop/fs/DU.java | 31 +++++++++++++++-----
.../test/java/org/apache/hadoop/fs/TestDU.java | 13 ++++----
3 files changed, 34 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/834d4217/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 7fd2320..a8a96be 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -32,6 +32,10 @@ Release 2.7.4 - UNRELEASED
HADOOP-14276. Add a nanosecond API to Time/Timer/FakeTimer.
(Erik Krogen via zhz)
+ HADOOP-12975. Add jitter to CachingGetSpaceUsed's thread
+ (Elliott Clark via Colin P. McCabe).
+ HADOOP-14408. Backport contributed by Erik Krogen.
+
OPTIMIZATIONS
HADOOP-14138. Remove S3A ref from META-INF service discovery, rely on
http://git-wip-us.apache.org/repos/asf/hadoop/blob/834d4217/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
index 5a4f526..51104a7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
@@ -17,10 +17,11 @@
*/
package org.apache.hadoop.fs;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.util.Shell;
import java.io.BufferedReader;
@@ -32,6 +33,9 @@ import java.util.concurrent.atomic.AtomicLong;
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Evolving
public class DU extends Shell {
+ static final String JITTER_KEY = "fs.getspaceused.jitterMillis";
+ static final long DEFAULT_JITTER = TimeUnit.MINUTES.toMillis(1);
+
private String dirPath;
private AtomicLong used = new AtomicLong();
@@ -39,7 +43,8 @@ public class DU extends Shell {
private Thread refreshUsed;
private IOException duException = null;
private long refreshInterval;
-
+ private final long jitter;
+
/**
* Keeps track of disk usage.
* @param path the path to check disk usage in
@@ -47,18 +52,23 @@ public class DU extends Shell {
* @throws IOException if we fail to refresh the disk usage
*/
public DU(File path, long interval) throws IOException {
- this(path, interval, -1L);
+ this(path, interval, 0L, -1L);
}
-
+
/**
* Keeps track of disk usage.
* @param path the path to check disk usage in
* @param interval refresh the disk usage at this interval
+ * @param jitter randomize the refresh interval timing by this amount; the
+ * actual interval will be randomly chosen between
+ * {@code interval-jitter} and {@code interval+jitter}
* @param initialUsed use this value until next refresh
* @throws IOException if we fail to refresh the disk usage
*/
- public DU(File path, long interval, long initialUsed) throws IOException {
+ public DU(File path, long interval, long jitter, long initialUsed)
+ throws IOException {
super(0);
+ this.jitter = jitter;
//we set the Shell interval to 0 so it will always run our command
//and use this one to set the thread sleep interval
@@ -93,7 +103,8 @@ public class DU extends Shell {
public DU(File path, Configuration conf, long initialUsed)
throws IOException {
this(path, conf.getLong(CommonConfigurationKeys.FS_DU_INTERVAL_KEY,
- CommonConfigurationKeys.FS_DU_INTERVAL_DEFAULT), initialUsed);
+ CommonConfigurationKeys.FS_DU_INTERVAL_DEFAULT),
+ conf.getLong(JITTER_KEY, DEFAULT_JITTER), initialUsed);
}
@@ -112,7 +123,13 @@ public class DU extends Shell {
while(shouldRun) {
try {
- Thread.sleep(refreshInterval);
+ long thisRefreshInterval = refreshInterval;
+ if (jitter > 0) {
+ // add/subtract the jitter.
+ thisRefreshInterval +=
+ ThreadLocalRandom.current().nextLong(-jitter, jitter);
+ }
+ Thread.sleep(thisRefreshInterval);
try {
//update the used variable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/834d4217/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java
index 3add2a7..e1943af 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java
@@ -25,7 +25,6 @@ import java.io.RandomAccessFile;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
/** This test makes sure that "DU" does not get to run on each call to getUsed */
public class TestDU extends TestCase {
@@ -78,8 +77,8 @@ public class TestDU extends TestCase {
createFile(file, writtenSize);
Thread.sleep(5000); // let the metadata updater catch up
-
- DU du = new DU(file, 10000);
+
+ DU du = new DU(file, 10000, 0, -1);
du.start();
long duSize = du.getUsed();
du.shutdown();
@@ -89,7 +88,7 @@ public class TestDU extends TestCase {
writtenSize <= (duSize + slack));
//test with 0 interval, will not launch thread
- du = new DU(file, 0);
+ du = new DU(file, 0, 1, -1);
du.start();
duSize = du.getUsed();
du.shutdown();
@@ -99,7 +98,7 @@ public class TestDU extends TestCase {
writtenSize <= (duSize + slack));
//test without launching thread
- du = new DU(file, 10000);
+ du = new DU(file, 10000, 0, -1);
duSize = du.getUsed();
assertTrue("Invalid on-disk size",
@@ -111,7 +110,7 @@ public class TestDU extends TestCase {
assertTrue(file.createNewFile());
Configuration conf = new Configuration();
conf.setLong(CommonConfigurationKeys.FS_DU_INTERVAL_KEY, 10000L);
- DU du = new DU(file, conf);
+ DU du = new DU(file, 10000L, 0, -1);
du.decDfsUsed(Long.MAX_VALUE);
long duSize = du.getUsed();
assertTrue(String.valueOf(duSize), duSize >= 0L);
@@ -120,7 +119,7 @@ public class TestDU extends TestCase {
public void testDUSetInitialValue() throws IOException {
File file = new File(DU_DIR, "dataX");
createFile(file, 8192);
- DU du = new DU(file, 3000, 1024);
+ DU du = new DU(file, 3000, 0, 1024);
du.start();
assertTrue("Initial usage setting not honored", du.getUsed() == 1024);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org