You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2021/06/20 05:36:55 UTC
[hadoop] branch branch-3.3 updated (e55d76e -> 92ade1f)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a change to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from e55d76e HDFS-13671. Namenode deletes large dir slowly caused by FoldedTreeSet#removeAndGet (#3113)
new 8d24148 HDFS-16078. Remove unused parameters for DatanodeManager.handleLifeline() (#3119). Contributed by tomscut.
new 92ade1f HDFS-16061. DFTestUtil.waitReplication can produce false positives (#3095). Contributed by Ahmed Hussein.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../server/blockmanagement/DatanodeManager.java | 6 +--
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 3 +-
.../java/org/apache/hadoop/hdfs/DFSTestUtil.java | 51 ++++++++++++----------
.../hdfs/server/balancer/TestBalancerRPCDelay.java | 8 +++-
4 files changed, 38 insertions(+), 30 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: HDFS-16078. Remove unused parameters for
DatanodeManager.handleLifeline() (#3119). Contributed by tomscut.
Posted by ay...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 8d241482bd3de948a900c4fd23e230066e8089e7
Author: litao <to...@gmail.com>
AuthorDate: Sun Jun 20 12:50:52 2021 +0800
HDFS-16078. Remove unused parameters for DatanodeManager.handleLifeline() (#3119). Contributed by tomscut.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Signed-off-by: Ayush Saxena <ay...@apache.org>
---
.../apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java | 6 ++----
.../java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 3 +--
2 files changed, 3 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 3a2ea65..eb85ce4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -1876,18 +1876,16 @@ public class DatanodeManager {
*
* @param nodeReg registration info for DataNode sending the lifeline
* @param reports storage reports from DataNode
- * @param blockPoolId block pool ID
* @param cacheCapacity cache capacity at DataNode
* @param cacheUsed cache used at DataNode
* @param xceiverCount estimated count of transfer threads running at DataNode
- * @param maxTransfers count of transfers running at DataNode
* @param failedVolumes count of failed volumes at DataNode
* @param volumeFailureSummary info on failed volumes at DataNode
* @throws IOException if there is an error
*/
public void handleLifeline(DatanodeRegistration nodeReg,
- StorageReport[] reports, String blockPoolId, long cacheCapacity,
- long cacheUsed, int xceiverCount, int maxTransfers, int failedVolumes,
+ StorageReport[] reports, long cacheCapacity,
+ long cacheUsed, int xceiverCount, int failedVolumes,
VolumeFailureSummary volumeFailureSummary) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Received handleLifeline from nodeReg = " + nodeReg);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index d2b2a83..54ce1f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4334,9 +4334,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
long cacheCapacity, long cacheUsed, int xceiverCount, int xmitsInProgress,
int failedVolumes, VolumeFailureSummary volumeFailureSummary)
throws IOException {
- int maxTransfer = blockManager.getMaxReplicationStreams() - xmitsInProgress;
blockManager.getDatanodeManager().handleLifeline(nodeReg, reports,
- getBlockPoolId(), cacheCapacity, cacheUsed, xceiverCount, maxTransfer,
+ cacheCapacity, cacheUsed, xceiverCount,
failedVolumes, volumeFailureSummary);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HDFS-16061. DFTestUtil.waitReplication can produce
false positives (#3095). Contributed by Ahmed Hussein.
Posted by ay...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 92ade1f6f9fa7591742307571a7b7be19576b678
Author: Ahmed Hussein <50...@users.noreply.github.com>
AuthorDate: Sat Jun 19 23:52:47 2021 -0500
HDFS-16061. DFTestUtil.waitReplication can produce false positives (#3095). Contributed by Ahmed Hussein.
Reviewed-by: Jim Brennan <jb...@apache.org>
Signed-off-by: Ayush Saxena <ay...@apache.org>
---
.../java/org/apache/hadoop/hdfs/DFSTestUtil.java | 51 ++++++++++++----------
.../hdfs/server/balancer/TestBalancerRPCDelay.java | 8 +++-
2 files changed, 35 insertions(+), 24 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 02dcb6d4..e9a732f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -799,41 +799,48 @@ public class DFSTestUtil {
/**
* Wait for the given file to reach the given replication factor.
- * @throws TimeoutException if we fail to sufficiently replicate the file
+ *
+ * @param fs the defined filesystem.
+ * @param fileName being written.
+ * @param replFactor desired replication
+ * @throws IOException getting block locations
+ * @throws InterruptedException during sleep
+ * @throws TimeoutException if 40 seconds passed before reaching the desired
+ * replication.
*/
- public static void waitReplication(FileSystem fs, Path fileName, short replFactor)
+ public static void waitReplication(FileSystem fs, Path fileName,
+ short replFactor)
throws IOException, InterruptedException, TimeoutException {
boolean correctReplFactor;
- final int ATTEMPTS = 40;
- int count = 0;
-
+ int attempt = 0;
do {
correctReplFactor = true;
+ if (attempt++ > 0) {
+ Thread.sleep(1000);
+ }
BlockLocation locs[] = fs.getFileBlockLocations(
- fs.getFileStatus(fileName), 0, Long.MAX_VALUE);
- count++;
- for (int j = 0; j < locs.length; j++) {
- String[] hostnames = locs[j].getNames();
+ fs.getFileStatus(fileName), 0, Long.MAX_VALUE);
+ for (int currLoc = 0; currLoc < locs.length; currLoc++) {
+ String[] hostnames = locs[currLoc].getNames();
if (hostnames.length != replFactor) {
+ LOG.info(
+ "Block {} of file {} has replication factor {} "
+ + "(desired {}); locations: {}",
+ currLoc, fileName, hostnames.length, replFactor,
+ Joiner.on(' ').join(hostnames));
correctReplFactor = false;
- System.out.println("Block " + j + " of file " + fileName
- + " has replication factor " + hostnames.length
- + " (desired " + replFactor + "); locations "
- + Joiner.on(' ').join(hostnames));
- Thread.sleep(1000);
break;
}
}
- if (correctReplFactor) {
- System.out.println("All blocks of file " + fileName
- + " verified to have replication factor " + replFactor);
- }
- } while (!correctReplFactor && count < ATTEMPTS);
+ } while (!correctReplFactor && attempt < 40);
- if (count == ATTEMPTS) {
- throw new TimeoutException("Timed out waiting for " + fileName +
- " to reach " + replFactor + " replicas");
+ if (!correctReplFactor) {
+ throw new TimeoutException("Timed out waiting for file ["
+ + fileName + "] to reach [" + replFactor + "] replicas");
}
+
+ LOG.info("All blocks of file {} verified to have replication factor {}",
+ fileName, replFactor);
}
/** delete directory and everything underneath it.*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerRPCDelay.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerRPCDelay.java
index 79c7f87..9752d65 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerRPCDelay.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerRPCDelay.java
@@ -20,13 +20,17 @@ package org.apache.hadoop.hdfs.server.balancer;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.Timeout;
/**
* The Balancer ensures that it disperses RPCs to the NameNode
* in order to avoid NN's RPC queue saturation.
*/
public class TestBalancerRPCDelay {
+ @Rule
+ public Timeout globalTimeout = Timeout.seconds(100);
private TestBalancer testBalancer;
@@ -43,12 +47,12 @@ public class TestBalancerRPCDelay {
}
}
- @Test(timeout=100000)
+ @Test
public void testBalancerRPCDelayQps3() throws Exception {
testBalancer.testBalancerRPCDelay(3);
}
- @Test(timeout=100000)
+ @Test
public void testBalancerRPCDelayQpsDefault() throws Exception {
testBalancer.testBalancerRPCDelay(
DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_MAX_QPS_DEFAULT);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org