You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by he...@apache.org on 2023/02/08 03:20:12 UTC
[hadoop] branch trunk updated (eb04ecd29d4 -> 3ba058a894b)
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from eb04ecd29d4 HDFS-16898. Make write lock fine-grain in method processCommandFromActor (#5330). Contributed by ZhangHB.
new 7e919212c46 Revert "HDFS-16898. Make write lock fine-grain in method processCommandFromActor (#5330). Contributed by ZhangHB."
new 3ba058a894b HDFS-16898. Remove write lock for processCommandFromActor of DataNode to reduce impact on heartbeat (#5330). Contributed by ZhangHB.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: Revert "HDFS-16898. Make write lock fine-grain in method processCommandFromActor (#5330). Contributed by ZhangHB."
Posted by he...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 7e919212c4672c99072dbecf8a1ec48fcb6de5f0
Author: He Xiaoqiao <he...@apache.org>
AuthorDate: Wed Feb 8 11:15:22 2023 +0800
Revert "HDFS-16898. Make write lock fine-grain in method processCommandFromActor (#5330). Contributed by ZhangHB."
This reverts commit eb04ecd29d4a38b1b00612cbb3a56ed3a34a9572.
---
.../hdfs/server/datanode/BPOfferService.java | 31 +++++++++-------------
.../hdfs/server/datanode/BPServiceActor.java | 2 +-
2 files changed, 14 insertions(+), 19 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index fdd66cb05d3..d660970c725 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -679,20 +679,15 @@ class BPOfferService {
actor.reRegister();
return false;
}
- boolean isActiveActor;
- InetSocketAddress nnSocketAddress;
- readLock();
+ writeLock();
try {
- isActiveActor = (actor == bpServiceToActive);
- nnSocketAddress = actor.getNNSocketAddress();
+ if (actor == bpServiceToActive) {
+ return processCommandFromActive(cmd, actor);
+ } else {
+ return processCommandFromStandby(cmd, actor);
+ }
} finally {
- readUnlock();
- }
-
- if (isActiveActor) {
- return processCommandFromActive(cmd, nnSocketAddress);
- } else {
- return processCommandFromStandby(cmd, nnSocketAddress);
+ writeUnlock();
}
}
@@ -720,7 +715,7 @@ class BPOfferService {
* @throws IOException
*/
private boolean processCommandFromActive(DatanodeCommand cmd,
- InetSocketAddress nnSocketAddress) throws IOException {
+ BPServiceActor actor) throws IOException {
final BlockCommand bcmd =
cmd instanceof BlockCommand? (BlockCommand)cmd: null;
final BlockIdCommand blockIdCmd =
@@ -773,7 +768,7 @@ class BPOfferService {
dn.finalizeUpgradeForPool(bp);
break;
case DatanodeProtocol.DNA_RECOVERBLOCK:
- String who = "NameNode at " + nnSocketAddress;
+ String who = "NameNode at " + actor.getNNSocketAddress();
dn.getBlockRecoveryWorker().recoverBlocks(who,
((BlockRecoveryCommand)cmd).getRecoveringBlocks());
break;
@@ -815,11 +810,11 @@ class BPOfferService {
* DNA_REGISTER which should be handled earlier itself.
*/
private boolean processCommandFromStandby(DatanodeCommand cmd,
- InetSocketAddress nnSocketAddress) throws IOException {
+ BPServiceActor actor) throws IOException {
switch(cmd.getAction()) {
case DatanodeProtocol.DNA_ACCESSKEYUPDATE:
LOG.info("DatanodeCommand action from standby NN {}: DNA_ACCESSKEYUPDATE",
- nnSocketAddress);
+ actor.getNNSocketAddress());
if (dn.isBlockTokenEnabled) {
dn.blockPoolTokenSecretManager.addKeys(
getBlockPoolId(),
@@ -836,11 +831,11 @@ class BPOfferService {
case DatanodeProtocol.DNA_UNCACHE:
case DatanodeProtocol.DNA_ERASURE_CODING_RECONSTRUCTION:
LOG.warn("Got a command from standby NN {} - ignoring command: {}",
- nnSocketAddress, cmd.getAction());
+ actor.getNNSocketAddress(), cmd.getAction());
break;
default:
LOG.warn("Unknown DatanodeCommand action: {} from standby NN {}",
- cmd.getAction(), nnSocketAddress);
+ cmd.getAction(), actor.getNNSocketAddress());
}
return true;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index e9f424604b4..35ab6193142 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -1499,7 +1499,7 @@ class BPServiceActor implements Runnable {
dn.getMetrics().addNumProcessedCommands(processCommandsMs);
}
if (processCommandsMs > dnConf.getProcessCommandsThresholdMs()) {
- LOG.warn("Took {} ms to process {} commands from NN",
+ LOG.info("Took {} ms to process {} commands from NN",
processCommandsMs, cmds.length);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HDFS-16898. Remove write lock for processCommandFromActor of DataNode to reduce impact on heartbeat (#5330). Contributed by ZhangHB.
Posted by he...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 3ba058a894b14c13651a52219b72eff48d07f4f4
Author: He Xiaoqiao <he...@apache.org>
AuthorDate: Wed Feb 8 11:19:07 2023 +0800
HDFS-16898. Remove write lock for processCommandFromActor of DataNode to reduce impact on heartbeat (#5330). Contributed by ZhangHB.
Reviewed-by: zhangshuyan <zq...@gmail.com>
Reviewed-by: Viraj Jasani <vj...@apache.org>
Signed-off-by: He Xiaoqiao <he...@apache.org>
---
.../hdfs/server/datanode/BPOfferService.java | 31 +++++++++++++---------
.../hdfs/server/datanode/BPServiceActor.java | 2 +-
2 files changed, 19 insertions(+), 14 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index d660970c725..fdd66cb05d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -679,15 +679,20 @@ class BPOfferService {
actor.reRegister();
return false;
}
- writeLock();
+ boolean isActiveActor;
+ InetSocketAddress nnSocketAddress;
+ readLock();
try {
- if (actor == bpServiceToActive) {
- return processCommandFromActive(cmd, actor);
- } else {
- return processCommandFromStandby(cmd, actor);
- }
+ isActiveActor = (actor == bpServiceToActive);
+ nnSocketAddress = actor.getNNSocketAddress();
} finally {
- writeUnlock();
+ readUnlock();
+ }
+
+ if (isActiveActor) {
+ return processCommandFromActive(cmd, nnSocketAddress);
+ } else {
+ return processCommandFromStandby(cmd, nnSocketAddress);
}
}
@@ -715,7 +720,7 @@ class BPOfferService {
* @throws IOException
*/
private boolean processCommandFromActive(DatanodeCommand cmd,
- BPServiceActor actor) throws IOException {
+ InetSocketAddress nnSocketAddress) throws IOException {
final BlockCommand bcmd =
cmd instanceof BlockCommand? (BlockCommand)cmd: null;
final BlockIdCommand blockIdCmd =
@@ -768,7 +773,7 @@ class BPOfferService {
dn.finalizeUpgradeForPool(bp);
break;
case DatanodeProtocol.DNA_RECOVERBLOCK:
- String who = "NameNode at " + actor.getNNSocketAddress();
+ String who = "NameNode at " + nnSocketAddress;
dn.getBlockRecoveryWorker().recoverBlocks(who,
((BlockRecoveryCommand)cmd).getRecoveringBlocks());
break;
@@ -810,11 +815,11 @@ class BPOfferService {
* DNA_REGISTER which should be handled earlier itself.
*/
private boolean processCommandFromStandby(DatanodeCommand cmd,
- BPServiceActor actor) throws IOException {
+ InetSocketAddress nnSocketAddress) throws IOException {
switch(cmd.getAction()) {
case DatanodeProtocol.DNA_ACCESSKEYUPDATE:
LOG.info("DatanodeCommand action from standby NN {}: DNA_ACCESSKEYUPDATE",
- actor.getNNSocketAddress());
+ nnSocketAddress);
if (dn.isBlockTokenEnabled) {
dn.blockPoolTokenSecretManager.addKeys(
getBlockPoolId(),
@@ -831,11 +836,11 @@ class BPOfferService {
case DatanodeProtocol.DNA_UNCACHE:
case DatanodeProtocol.DNA_ERASURE_CODING_RECONSTRUCTION:
LOG.warn("Got a command from standby NN {} - ignoring command: {}",
- actor.getNNSocketAddress(), cmd.getAction());
+ nnSocketAddress, cmd.getAction());
break;
default:
LOG.warn("Unknown DatanodeCommand action: {} from standby NN {}",
- cmd.getAction(), actor.getNNSocketAddress());
+ cmd.getAction(), nnSocketAddress);
}
return true;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 35ab6193142..e9f424604b4 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -1499,7 +1499,7 @@ class BPServiceActor implements Runnable {
dn.getMetrics().addNumProcessedCommands(processCommandsMs);
}
if (processCommandsMs > dnConf.getProcessCommandsThresholdMs()) {
- LOG.info("Took {} ms to process {} commands from NN",
+ LOG.warn("Took {} ms to process {} commands from NN",
processCommandsMs, cmds.length);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org