You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by fe...@apache.org on 2024/04/09 01:35:46 UTC
(hadoop) branch HDFS-17384 updated: HDFS-17411. [FGL] Client RPCs involving snapshot support fine-grained lock (#6714)
This is an automated email from the ASF dual-hosted git repository.
ferhui pushed a commit to branch HDFS-17384
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/HDFS-17384 by this push:
new 62685b61c19a HDFS-17411. [FGL] Client RPCs involving snapshot support fine-grained lock (#6714)
62685b61c19a is described below
commit 62685b61c19a4f061ef3963f069a150c6d885c3d
Author: ZanderXu <za...@apache.org>
AuthorDate: Tue Apr 9 09:35:36 2024 +0800
HDFS-17411. [FGL] Client RPCs involving snapshot support fine-grained lock (#6714)
---
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 45 ++++++++++++----------
.../namenode/snapshot/SnapshotDeletionGc.java | 5 ++-
.../hdfs/server/namenode/TestLeaseManager.java | 3 ++
3 files changed, 30 insertions(+), 23 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 647110c7601b..ddcf23ce55f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -7161,13 +7161,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkOperation(OperationCategory.WRITE);
final String operationName = "allowSnapshot";
checkSuperuserPrivilege(operationName, path);
- writeLock();
+ writeLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot allow snapshot for " + path);
FSDirSnapshotOp.allowSnapshot(dir, snapshotManager, path);
} finally {
- writeUnlock(operationName, getLockReportInfoSupplier(path));
+ writeUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(path));
}
getEditLog().logSync();
logAuditEvent(true, operationName, path, null, null);
@@ -7178,13 +7178,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkOperation(OperationCategory.WRITE);
final String operationName = "disallowSnapshot";
checkSuperuserPrivilege(operationName, path);
- writeLock();
+ writeLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot disallow snapshot for " + path);
FSDirSnapshotOp.disallowSnapshot(dir, snapshotManager, path);
} finally {
- writeUnlock(operationName, getLockReportInfoSupplier(path));
+ writeUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(path));
}
getEditLog().logSync();
logAuditEvent(true, operationName, path, null, null);
@@ -7203,14 +7203,15 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final FSPermissionChecker pc = getPermissionChecker();
FSPermissionChecker.setOperationType(operationName);
try {
- writeLock();
+ writeLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot create snapshot for " + snapshotRoot);
snapshotPath = FSDirSnapshotOp.createSnapshot(dir, pc,
snapshotManager, snapshotRoot, snapshotName, logRetryCache);
} finally {
- writeUnlock(operationName, getLockReportInfoSupplier(snapshotRoot));
+ writeUnlock(FSNamesystemLockMode.FS, operationName,
+ getLockReportInfoSupplier(snapshotRoot));
}
} catch (AccessControlException ace) {
logAuditEvent(false, operationName, snapshotRoot);
@@ -7240,15 +7241,15 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final FSPermissionChecker pc = getPermissionChecker();
FSPermissionChecker.setOperationType(operationName);
try {
- writeLock();
+ writeLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot rename snapshot for " + path);
FSDirSnapshotOp.renameSnapshot(dir, pc, snapshotManager, path,
snapshotOldName, snapshotNewName, logRetryCache);
} finally {
- writeUnlock(operationName, getLockReportInfoSupplier(oldSnapshotRoot,
- newSnapshotRoot));
+ writeUnlock(FSNamesystemLockMode.FS, operationName,
+ getLockReportInfoSupplier(oldSnapshotRoot, newSnapshotRoot));
}
} catch (AccessControlException ace) {
logAuditEvent(false, operationName, oldSnapshotRoot,
@@ -7275,13 +7276,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final FSPermissionChecker pc = getPermissionChecker();
FSPermissionChecker.setOperationType(operationName);
try {
- readLock();
+ readLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.READ);
status = FSDirSnapshotOp.getSnapshottableDirListing(dir, pc,
snapshotManager);
} finally {
- readUnlock(operationName, getLockReportInfoSupplier(null));
+ readUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(null));
}
} catch (AccessControlException ace) {
logAuditEvent(false, operationName, null, null, null);
@@ -7306,14 +7307,14 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final FSPermissionChecker pc = getPermissionChecker();
FSPermissionChecker.setOperationType(operationName);
try {
- readLock();
+ readLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.READ);
status = FSDirSnapshotOp.getSnapshotListing(dir, pc, snapshotManager,
snapshotRoot);
success = true;
} finally {
- readUnlock(operationName, getLockReportInfoSupplier(null));
+ readUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(null));
}
} catch (AccessControlException ace) {
logAuditEvent(success, operationName, snapshotRoot);
@@ -7351,14 +7352,14 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
FSPermissionChecker.setOperationType(operationName);
long actualTime = Time.monotonicNow();
try {
- readLock();
+ readLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.READ);
diffs = FSDirSnapshotOp.getSnapshotDiffReport(dir, pc, snapshotManager,
path, fromSnapshot, toSnapshot);
} finally {
- readUnlock(operationName, getLockReportInfoSupplier(fromSnapshotRoot,
- toSnapshotRoot));
+ readUnlock(FSNamesystemLockMode.FS, operationName,
+ getLockReportInfoSupplier(fromSnapshotRoot, toSnapshotRoot));
}
} catch (AccessControlException ace) {
logAuditEvent(false, operationName, fromSnapshotRoot,
@@ -7425,7 +7426,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final FSPermissionChecker pc = getPermissionChecker();
FSPermissionChecker.setOperationType(operationName);
try {
- readLock();
+ readLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.READ);
diffs = FSDirSnapshotOp
@@ -7433,8 +7434,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
fromSnapshot, toSnapshot, startPath, index,
snapshotDiffReportLimit);
} finally {
- readUnlock(operationName, getLockReportInfoSupplier(fromSnapshotRoot,
- toSnapshotRoot));
+ readUnlock(FSNamesystemLockMode.FS, operationName,
+ getLockReportInfoSupplier(fromSnapshotRoot, toSnapshotRoot));
}
} catch (AccessControlException ace) {
logAuditEvent(false, operationName, fromSnapshotRoot, toSnapshotRoot,
@@ -7462,7 +7463,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
FSPermissionChecker.setOperationType(operationName);
checkOperation(OperationCategory.WRITE);
try {
- writeLock();
+ // It involves Block while collecting blocks to be deleted.
+ writeLock(FSNamesystemLockMode.GLOBAL);
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot delete snapshot for " + snapshotRoot);
@@ -7470,7 +7472,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
blocksToBeDeleted = FSDirSnapshotOp.deleteSnapshot(dir, pc,
snapshotManager, snapshotRoot, snapshotName, logRetryCache);
} finally {
- writeUnlock(operationName, getLockReportInfoSupplier(rootPath));
+ writeUnlock(FSNamesystemLockMode.GLOBAL, operationName,
+ getLockReportInfoSupplier(rootPath));
}
} catch (AccessControlException ace) {
logAuditEvent(false, operationName, rootPath, null, null);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java
index c62455c724fe..5b509a7d85f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,14 +73,14 @@ public class SnapshotDeletionGc {
private void gcDeletedSnapshot(String name) {
final Snapshot.Root deleted;
- namesystem.readLock();
+ namesystem.readLock(FSNamesystemLockMode.FS);
try {
deleted = namesystem.getSnapshotManager().chooseDeletedSnapshot();
} catch (Throwable e) {
LOG.error("Failed to chooseDeletedSnapshot", e);
throw e;
} finally {
- namesystem.readUnlock("gcDeletedSnapshot");
+ namesystem.readUnlock(FSNamesystemLockMode.FS, "gcDeletedSnapshot");
}
if (deleted == null) {
LOG.trace("{}: no snapshots are marked as deleted.", name);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
index 2759b9e20d7e..6be34537a050 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.util.Lists;
@@ -466,6 +467,8 @@ public class TestLeaseManager {
when(fsn.isRunning()).thenReturn(true);
when(fsn.hasReadLock()).thenReturn(true);
when(fsn.hasWriteLock()).thenReturn(true);
+ when(fsn.hasReadLock(FSNamesystemLockMode.FS)).thenReturn(true);
+ when(fsn.hasWriteLock(FSNamesystemLockMode.FS)).thenReturn(true);
when(fsn.getFSDirectory()).thenReturn(dir);
when(fsn.getMaxLockHoldToReleaseLeaseMs()).thenReturn(maxLockHoldToReleaseLeaseMs);
return fsn;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org