You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2015/02/03 20:23:46 UTC
hadoop git commit: HDFS-7721. The HDFS BlockScanner may run fast
during the first hour (cmccabe)
Repository: hadoop
Updated Branches:
refs/heads/trunk c89977f89 -> 115428176
HDFS-7721. The HDFS BlockScanner may run fast during the first hour (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/11542817
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/11542817
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/11542817
Branch: refs/heads/trunk
Commit: 115428176e1d919fe7d54d01b34dfda57d1b3950
Parents: c89977f
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Feb 3 11:05:31 2015 -0800
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue Feb 3 11:05:31 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hdfs/server/datanode/VolumeScanner.java | 52 ++++++++++++++------
.../hdfs/server/datanode/TestBlockScanner.java | 36 ++++++++------
3 files changed, 61 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/11542817/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2deff81..d6283b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -278,6 +278,9 @@ Trunk (Unreleased)
HDFS-7320. The appearance of hadoop-hdfs-httpfs site docs is inconsistent
(Masatake Iwasaki via aw)
+ HDFS-7721. The HDFS BlockScanner may run fast during the first hour
+ (cmccabe)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/11542817/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
index 781b4d3..ce0a875 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
@@ -128,6 +128,11 @@ public class VolumeScanner extends Thread {
private boolean stopping = false;
/**
+ * The monotonic minute that the volume scanner was started on.
+ */
+ private long startMinute = 0;
+
+ /**
* The current minute, in monotonic terms.
*/
private long curMinute = 0;
@@ -297,18 +302,18 @@ public class VolumeScanner extends Thread {
private void expireOldScannedBytesRecords(long monotonicMs) {
long newMinute =
TimeUnit.MINUTES.convert(monotonicMs, TimeUnit.MILLISECONDS);
- newMinute = newMinute % MINUTES_PER_HOUR;
if (curMinute == newMinute) {
return;
}
// If a minute or more has gone past since we last updated the scannedBytes
// array, zero out the slots corresponding to those minutes.
for (long m = curMinute + 1; m <= newMinute; m++) {
- LOG.trace("{}: updateScannedBytes is zeroing out slot {}. " +
- "curMinute = {}; newMinute = {}", this, m % MINUTES_PER_HOUR,
- curMinute, newMinute);
- scannedBytesSum -= scannedBytes[(int)(m % MINUTES_PER_HOUR)];
- scannedBytes[(int)(m % MINUTES_PER_HOUR)] = 0;
+ int slotIdx = (int)(m % MINUTES_PER_HOUR);
+ LOG.trace("{}: updateScannedBytes is zeroing out slotIdx {}. " +
+ "curMinute = {}; newMinute = {}", this, slotIdx,
+ curMinute, newMinute);
+ scannedBytesSum -= scannedBytes[slotIdx];
+ scannedBytes[slotIdx] = 0;
}
curMinute = newMinute;
}
@@ -425,14 +430,28 @@ public class VolumeScanner extends Thread {
}
@VisibleForTesting
- static boolean calculateShouldScan(long targetBytesPerSec,
- long scannedBytesSum) {
- long effectiveBytesPerSec =
- scannedBytesSum / (SECONDS_PER_MINUTE * MINUTES_PER_HOUR);
+ static boolean calculateShouldScan(String storageId, long targetBytesPerSec,
+ long scannedBytesSum, long startMinute, long curMinute) {
+ long runMinutes = curMinute - startMinute;
+ long effectiveBytesPerSec;
+ if (runMinutes <= 0) {
+ // avoid division by zero
+ effectiveBytesPerSec = scannedBytesSum;
+ } else {
+ if (runMinutes > MINUTES_PER_HOUR) {
+ // we only keep an hour's worth of rate information
+ runMinutes = MINUTES_PER_HOUR;
+ }
+ effectiveBytesPerSec = scannedBytesSum /
+ (SECONDS_PER_MINUTE * runMinutes);
+ }
+
boolean shouldScan = effectiveBytesPerSec <= targetBytesPerSec;
- LOG.trace("calculateShouldScan: effectiveBytesPerSec = {}, and " +
- "targetBytesPerSec = {}. shouldScan = {}",
- effectiveBytesPerSec, targetBytesPerSec, shouldScan);
+ LOG.trace("{}: calculateShouldScan: effectiveBytesPerSec = {}, and " +
+ "targetBytesPerSec = {}. startMinute = {}, curMinute = {}, " +
+ "shouldScan = {}",
+ storageId, effectiveBytesPerSec, targetBytesPerSec,
+ startMinute, curMinute, shouldScan);
return shouldScan;
}
@@ -450,7 +469,8 @@ public class VolumeScanner extends Thread {
long monotonicMs = Time.monotonicNow();
expireOldScannedBytesRecords(monotonicMs);
- if (!calculateShouldScan(conf.targetBytesPerSec, scannedBytesSum)) {
+ if (!calculateShouldScan(volume.getStorageID(), conf.targetBytesPerSec,
+ scannedBytesSum, startMinute, curMinute)) {
// If neededBytesPerSec is too low, then wait few seconds for some old
// scannedBytes records to expire.
return 30000L;
@@ -533,6 +553,10 @@ public class VolumeScanner extends Thread {
@Override
public void run() {
+ // Record the minute on which the scanner started.
+ this.startMinute =
+ TimeUnit.MINUTES.convert(Time.monotonicNow(), TimeUnit.MILLISECONDS);
+ this.curMinute = startMinute;
try {
LOG.trace("{}: thread starting.", this);
resultHandler.setup(this);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/11542817/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
index 7eaa2bf..b727263 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
@@ -431,13 +431,6 @@ public class TestBlockScanner {
info.shouldRun = true;
info.notify();
}
- Thread.sleep(5000);
- synchronized (info) {
- long endMs = Time.monotonicNow();
- // Should scan no more than one block a second.
- long maxBlocksScanned = ((endMs + 999 - startMs) / 1000);
- assertTrue(info.blocksScanned < maxBlocksScanned);
- }
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
@@ -446,6 +439,17 @@ public class TestBlockScanner {
}
}
}, 1, 30000);
+ Thread.sleep(2000);
+ synchronized (info) {
+ long endMs = Time.monotonicNow();
+ // Should scan no more than one block a second.
+ long seconds = ((endMs + 999 - startMs) / 1000);
+ long maxBlocksScanned = seconds * 1;
+ assertTrue("The number of blocks scanned is too large. Scanned " +
+ info.blocksScanned + " blocks; only expected to scan at most " +
+ maxBlocksScanned + " in " + seconds + " seconds.",
+ info.blocksScanned <= maxBlocksScanned);
+ }
ctx.close();
}
@@ -657,24 +661,24 @@ public class TestBlockScanner {
public void testCalculateNeededBytesPerSec() throws Exception {
// If we didn't check anything the last hour, we should scan now.
Assert.assertTrue(
- VolumeScanner.calculateShouldScan(100, 0));
+ VolumeScanner.calculateShouldScan("test", 100, 0, 0, 60));
// If, on average, we checked 101 bytes/s checked during the last hour,
// stop checking now.
- Assert.assertFalse(
- VolumeScanner.calculateShouldScan(100, 101 * 3600));
+ Assert.assertFalse(VolumeScanner.
+ calculateShouldScan("test", 100, 101 * 3600, 1000, 5000));
// Target is 1 byte / s, but we didn't scan anything in the last minute.
// Should scan now.
- Assert.assertTrue(
- VolumeScanner.calculateShouldScan(1, 3540));
+ Assert.assertTrue(VolumeScanner.
+ calculateShouldScan("test", 1, 3540, 0, 60));
// Target is 1000000 byte / s, but we didn't scan anything in the last
// minute. Should scan now.
- Assert.assertTrue(
- VolumeScanner.calculateShouldScan(100000L, 354000000L));
+ Assert.assertTrue(VolumeScanner.
+ calculateShouldScan("test", 100000L, 354000000L, 0, 60));
- Assert.assertFalse(
- VolumeScanner.calculateShouldScan(100000L, 365000000L));
+ Assert.assertFalse(VolumeScanner.
+ calculateShouldScan("test", 100000L, 365000000L, 0, 60));
}
}