You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2021/06/20 05:36:52 UTC
[hadoop] branch branch-3.2 updated (c14f3e4 -> dc46cec)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a change to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from c14f3e4 HDFS-13671. Namenode deletes large dir slowly caused by FoldedTreeSet#removeAndGet (#3114)
new 00bd7b2 HDFS-16078. Remove unused parameters for DatanodeManager.handleLifeline() (#3119). Contributed by tomscut.
new dc46cec HDFS-16061. DFTestUtil.waitReplication can produce false positives (#3095). Contributed by Ahmed Hussein.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../server/blockmanagement/DatanodeManager.java | 6 +--
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 3 +-
.../java/org/apache/hadoop/hdfs/DFSTestUtil.java | 51 ++++++++++++----------
.../hdfs/server/balancer/TestBalancerRPCDelay.java | 8 +++-
4 files changed, 38 insertions(+), 30 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: HDFS-16078. Remove unused parameters for
DatanodeManager.handleLifeline() (#3119). Contributed by tomscut.
Posted by ay...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 00bd7b2aa4809b42d24bfe60db35ba5ed4386c3c
Author: litao <to...@gmail.com>
AuthorDate: Sun Jun 20 12:50:52 2021 +0800
HDFS-16078. Remove unused parameters for DatanodeManager.handleLifeline() (#3119). Contributed by tomscut.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Signed-off-by: Ayush Saxena <ay...@apache.org>
---
.../apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java | 6 ++----
.../java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 3 +--
2 files changed, 3 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 771ad45..9104411 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -1776,18 +1776,16 @@ public class DatanodeManager {
*
* @param nodeReg registration info for DataNode sending the lifeline
* @param reports storage reports from DataNode
- * @param blockPoolId block pool ID
* @param cacheCapacity cache capacity at DataNode
* @param cacheUsed cache used at DataNode
* @param xceiverCount estimated count of transfer threads running at DataNode
- * @param maxTransfers count of transfers running at DataNode
* @param failedVolumes count of failed volumes at DataNode
* @param volumeFailureSummary info on failed volumes at DataNode
* @throws IOException if there is an error
*/
public void handleLifeline(DatanodeRegistration nodeReg,
- StorageReport[] reports, String blockPoolId, long cacheCapacity,
- long cacheUsed, int xceiverCount, int maxTransfers, int failedVolumes,
+ StorageReport[] reports, long cacheCapacity,
+ long cacheUsed, int xceiverCount, int failedVolumes,
VolumeFailureSummary volumeFailureSummary) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Received handleLifeline from nodeReg = " + nodeReg);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 09c1bf0..7a7d072 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4020,9 +4020,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
long cacheCapacity, long cacheUsed, int xceiverCount, int xmitsInProgress,
int failedVolumes, VolumeFailureSummary volumeFailureSummary)
throws IOException {
- int maxTransfer = blockManager.getMaxReplicationStreams() - xmitsInProgress;
blockManager.getDatanodeManager().handleLifeline(nodeReg, reports,
- getBlockPoolId(), cacheCapacity, cacheUsed, xceiverCount, maxTransfer,
+ cacheCapacity, cacheUsed, xceiverCount,
failedVolumes, volumeFailureSummary);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HDFS-16061. DFTestUtil.waitReplication can produce
false positives (#3095). Contributed by Ahmed Hussein.
Posted by ay...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit dc46cec277006c21ed812353fbd5e6c7024d4ee1
Author: Ahmed Hussein <50...@users.noreply.github.com>
AuthorDate: Sat Jun 19 23:52:47 2021 -0500
HDFS-16061. DFTestUtil.waitReplication can produce false positives (#3095). Contributed by Ahmed Hussein.
Reviewed-by: Jim Brennan <jb...@apache.org>
Signed-off-by: Ayush Saxena <ay...@apache.org>
---
.../java/org/apache/hadoop/hdfs/DFSTestUtil.java | 51 ++++++++++++----------
.../hdfs/server/balancer/TestBalancerRPCDelay.java | 8 +++-
2 files changed, 35 insertions(+), 24 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index b0c9aca..88a9483 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -779,41 +779,48 @@ public class DFSTestUtil {
/**
* Wait for the given file to reach the given replication factor.
- * @throws TimeoutException if we fail to sufficiently replicate the file
+ *
+ * @param fs the defined filesystem.
+ * @param fileName being written.
+ * @param replFactor desired replication
+ * @throws IOException getting block locations
+ * @throws InterruptedException during sleep
+ * @throws TimeoutException if 40 seconds passed before reaching the desired
+ * replication.
*/
- public static void waitReplication(FileSystem fs, Path fileName, short replFactor)
+ public static void waitReplication(FileSystem fs, Path fileName,
+ short replFactor)
throws IOException, InterruptedException, TimeoutException {
boolean correctReplFactor;
- final int ATTEMPTS = 40;
- int count = 0;
-
+ int attempt = 0;
do {
correctReplFactor = true;
+ if (attempt++ > 0) {
+ Thread.sleep(1000);
+ }
BlockLocation locs[] = fs.getFileBlockLocations(
- fs.getFileStatus(fileName), 0, Long.MAX_VALUE);
- count++;
- for (int j = 0; j < locs.length; j++) {
- String[] hostnames = locs[j].getNames();
+ fs.getFileStatus(fileName), 0, Long.MAX_VALUE);
+ for (int currLoc = 0; currLoc < locs.length; currLoc++) {
+ String[] hostnames = locs[currLoc].getNames();
if (hostnames.length != replFactor) {
+ LOG.info(
+ "Block {} of file {} has replication factor {} "
+ + "(desired {}); locations: {}",
+ currLoc, fileName, hostnames.length, replFactor,
+ Joiner.on(' ').join(hostnames));
correctReplFactor = false;
- System.out.println("Block " + j + " of file " + fileName
- + " has replication factor " + hostnames.length
- + " (desired " + replFactor + "); locations "
- + Joiner.on(' ').join(hostnames));
- Thread.sleep(1000);
break;
}
}
- if (correctReplFactor) {
- System.out.println("All blocks of file " + fileName
- + " verified to have replication factor " + replFactor);
- }
- } while (!correctReplFactor && count < ATTEMPTS);
+ } while (!correctReplFactor && attempt < 40);
- if (count == ATTEMPTS) {
- throw new TimeoutException("Timed out waiting for " + fileName +
- " to reach " + replFactor + " replicas");
+ if (!correctReplFactor) {
+ throw new TimeoutException("Timed out waiting for file ["
+ + fileName + "] to reach [" + replFactor + "] replicas");
}
+
+ LOG.info("All blocks of file {} verified to have replication factor {}",
+ fileName, replFactor);
}
/** delete directory and everything underneath it.*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerRPCDelay.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerRPCDelay.java
index 79c7f87..9752d65 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerRPCDelay.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerRPCDelay.java
@@ -20,13 +20,17 @@ package org.apache.hadoop.hdfs.server.balancer;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.Timeout;
/**
* The Balancer ensures that it disperses RPCs to the NameNode
* in order to avoid NN's RPC queue saturation.
*/
public class TestBalancerRPCDelay {
+ @Rule
+ public Timeout globalTimeout = Timeout.seconds(100);
private TestBalancer testBalancer;
@@ -43,12 +47,12 @@ public class TestBalancerRPCDelay {
}
}
- @Test(timeout=100000)
+ @Test
public void testBalancerRPCDelayQps3() throws Exception {
testBalancer.testBalancerRPCDelay(3);
}
- @Test(timeout=100000)
+ @Test
public void testBalancerRPCDelayQpsDefault() throws Exception {
testBalancer.testBalancerRPCDelay(
DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_MAX_QPS_DEFAULT);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org