You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by za...@apache.org on 2025/01/23 07:40:34 UTC
(hadoop) 14/28: HDFS-17412. [FGL] Client RPCs involving maintenance supports fine-grained lock (#6667)
This is an automated email from the ASF dual-hosted git repository.
zanderxu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 03603efd74b124d77f8efc1c158f1430a4e631ca
Author: ZanderXu <za...@apache.org>
AuthorDate: Wed Mar 27 17:07:38 2024 +0800
HDFS-17412. [FGL] Client RPCs involving maintenance supports fine-grained lock (#6667)
---
.../hdfs/server/blockmanagement/BlockManager.java | 2 +-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 42 ++++++++++++----------
2 files changed, 24 insertions(+), 20 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 1e98290c78b..4d405443290 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -861,7 +861,7 @@ public void refreshBlockPlacementPolicy(Configuration conf) {
/** Dump meta data to out. */
public void metaSave(PrintWriter out) {
- assert namesystem.hasReadLock(); // TODO: block manager read lock and NS write lock
+ assert namesystem.hasReadLock(FSNamesystemLockMode.BM);
final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
datanodeManager.fetchDatanodes(live, dead, false);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 8570b053dc9..2a27e8c0273 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2001,7 +2001,7 @@ void metaSave(String filename) throws IOException {
String operationName = "metaSave";
checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.READ);
- readLock();
+ readLock(FSNamesystemLockMode.GLOBAL);
try {
checkOperation(OperationCategory.READ);
synchronized(metaSaveLock) {
@@ -2014,13 +2014,15 @@ void metaSave(String filename) throws IOException {
out.close();
}
} finally {
- readUnlock(operationName, getLockReportInfoSupplier(null));
+ readUnlock(FSNamesystemLockMode.GLOBAL, operationName, getLockReportInfoSupplier(null));
}
logAuditEvent(true, operationName, null);
}
private void metaSave(PrintWriter out) {
- assert hasReadLock();
+ // TODO: Change to hasReadLock(FSNamesystemLockMode.BM)
+ assert hasReadLock(FSNamesystemLockMode.GLOBAL);
+ // Normally FSReadLock is needed here, but I think thread-safe is unnecessary here.
long totalInodes = this.dir.totalInodes();
long totalBlocks = this.getBlocksTotal();
out.println(totalInodes + " files and directories, " + totalBlocks
@@ -5009,14 +5011,14 @@ DatanodeInfo[] slowDataNodesReport() throws IOException {
String operationName = "slowDataNodesReport";
DatanodeInfo[] datanodeInfos;
checkOperation(OperationCategory.UNCHECKED);
- readLock();
+ readLock(FSNamesystemLockMode.BM);
try {
checkOperation(OperationCategory.UNCHECKED);
final DatanodeManager dm = getBlockManager().getDatanodeManager();
final List<DatanodeDescriptor> results = dm.getAllSlowDataNodes();
datanodeInfos = getDatanodeInfoFromDescriptors(results);
} finally {
- readUnlock(operationName, getLockReportInfoSupplier(null));
+ readUnlock(FSNamesystemLockMode.BM, operationName, getLockReportInfoSupplier(null));
}
logAuditEvent(true, operationName, null);
return datanodeInfos;
@@ -5037,14 +5039,14 @@ DatanodeInfo[] datanodeReport(final DatanodeReportType type)
DatanodeInfo[] arr;
checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.UNCHECKED);
- readLock();
+ readLock(FSNamesystemLockMode.BM);
try {
checkOperation(OperationCategory.UNCHECKED);
final DatanodeManager dm = getBlockManager().getDatanodeManager();
final List<DatanodeDescriptor> results = dm.getDatanodeListForReport(type);
arr = getDatanodeInfoFromDescriptors(results);
} finally {
- readUnlock(operationName, getLockReportInfoSupplier(null));
+ readUnlock(FSNamesystemLockMode.BM, operationName, getLockReportInfoSupplier(null));
}
logAuditEvent(true, operationName, null);
return arr;
@@ -5056,13 +5058,13 @@ DatanodeStorageReport[] getDatanodeStorageReport(final DatanodeReportType type
DatanodeStorageReport[] reports;
checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.UNCHECKED);
- readLock();
+ readLock(FSNamesystemLockMode.BM);
try {
checkOperation(OperationCategory.UNCHECKED);
final DatanodeManager dm = getBlockManager().getDatanodeManager();
reports = dm.getDatanodeStorageReport(type);
} finally {
- readUnlock(operationName, getLockReportInfoSupplier(null));
+ readUnlock(FSNamesystemLockMode.BM, operationName, getLockReportInfoSupplier(null));
}
logAuditEvent(true, operationName, null);
return reports;
@@ -5081,7 +5083,9 @@ boolean saveNamespace(final long timeWindow, final long txGap)
boolean saved = false;
cpLock(); // Block if a checkpointing is in progress on standby.
- readLock();
+ // TODO: MileStone2 should change this readLock() to writeLock(FSNamesystemLockMode.FS)
+ // since all directory-tree modification operations will just hold the FSReadLock.
+ readLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.UNCHECKED);
@@ -5091,7 +5095,7 @@ boolean saveNamespace(final long timeWindow, final long txGap)
}
saved = getFSImage().saveNamespace(timeWindow, txGap, this);
} finally {
- readUnlock(operationName, getLockReportInfoSupplier(null));
+ readUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(null));
cpUnlock();
}
if (saved) {
@@ -5113,7 +5117,7 @@ boolean restoreFailedStorage(String arg) throws IOException {
checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.UNCHECKED);
cpLock(); // Block if a checkpointing is in progress on standby.
- writeLock();
+ writeLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.UNCHECKED);
@@ -5125,7 +5129,7 @@ boolean restoreFailedStorage(String arg) throws IOException {
getNNStorage().setRestoreFailedStorage(val);
}
} finally {
- writeUnlock(operationName, getLockReportInfoSupplier(null));
+ writeUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(null));
cpUnlock();
}
logAuditEvent(true, operationName, null);
@@ -5141,12 +5145,12 @@ void finalizeUpgrade() throws IOException {
checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.UNCHECKED);
cpLock(); // Block if a checkpointing is in progress on standby.
- writeLock();
+ writeLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.UNCHECKED);
getFSImage().finalizeUpgrade(this.isHaEnabled() && inActiveState());
} finally {
- writeUnlock(operationName, getLockReportInfoSupplier(null));
+ writeUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(null));
cpUnlock();
}
logAuditEvent(true, operationName, null);
@@ -7553,7 +7557,7 @@ RollingUpgradeInfo queryRollingUpgrade() throws IOException {
final String operationName = "queryRollingUpgrade";
checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.READ);
- readLock();
+ readLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.READ);
if (!isRollingUpgrade()) {
@@ -7563,7 +7567,7 @@ RollingUpgradeInfo queryRollingUpgrade() throws IOException {
boolean hasRollbackImage = this.getFSImage().hasRollbackFSImage();
rollingUpgradeInfo.setCreatedRollbackImages(hasRollbackImage);
} finally {
- readUnlock(operationName, getLockReportInfoSupplier(null));
+ readUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(null));
}
logAuditEvent(true, operationName, null, null, null);
return rollingUpgradeInfo;
@@ -7573,7 +7577,7 @@ RollingUpgradeInfo startRollingUpgrade() throws IOException {
final String operationName = "startRollingUpgrade";
checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.WRITE);
- writeLock();
+ writeLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.WRITE);
if (isRollingUpgrade()) {
@@ -7593,7 +7597,7 @@ RollingUpgradeInfo startRollingUpgrade() throws IOException {
getFSImage().rollEditLog(getEffectiveLayoutVersion());
}
} finally {
- writeUnlock(operationName, getLockReportInfoSupplier(null));
+ writeUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(null));
}
getEditLog().logSync();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org