You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by fe...@apache.org on 2024/04/10 02:08:33 UTC
(hadoop) branch HDFS-17384 updated: HDFS-17445. [FGL] some operations support fine-grained locking (#6715)
This is an automated email from the ASF dual-hosted git repository.
ferhui pushed a commit to branch HDFS-17384
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/HDFS-17384 by this push:
new 27ed48ef7f04 HDFS-17445. [FGL] some operations support fine-grained locking (#6715)
27ed48ef7f04 is described below
commit 27ed48ef7f04195efa5390a1133cb076ec86ef7b
Author: ZanderXu <za...@apache.org>
AuthorDate: Wed Apr 10 10:08:24 2024 +0800
HDFS-17445. [FGL] some operations support fine-grained locking (#6715)
---
.../hdfs/server/blockmanagement/BlockManager.java | 10 +++---
.../hadoop/hdfs/server/namenode/BackupImage.java | 5 +--
.../hdfs/server/namenode/FSDirErasureCodingOp.java | 2 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 2 +-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 40 +++++++++++-----------
.../hdfs/server/namenode/FSPermissionChecker.java | 15 ++++----
.../hdfs/server/namenode/FSTreeTraverser.java | 7 ++--
.../hdfs/server/namenode/FsImageValidation.java | 5 +--
.../hadoop/hdfs/server/namenode/LeaseManager.java | 12 +++----
.../hadoop/hdfs/server/namenode/NameNode.java | 25 +++++++-------
.../hadoop/hdfs/server/namenode/NamenodeFsck.java | 10 +++---
.../hdfs/server/namenode/SecondaryNameNode.java | 5 +--
.../hdfs/server/namenode/TestLeaseManager.java | 2 ++
13 files changed, 75 insertions(+), 65 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 996d09d946f3..b8ed9000c76d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2788,7 +2788,7 @@ public class BlockManager implements BlockStatsMXBean {
* list of blocks that need to be removed from blocksMap
*/
public void removeBlocksAndUpdateSafemodeTotal(BlocksMapUpdateInfo blocks) {
- assert namesystem.hasWriteLock();
+ assert namesystem.hasWriteLock(FSNamesystemLockMode.BM);
// In the case that we are a Standby tailing edits from the
// active while in safe-mode, we need to track the total number
// of blocks and safe blocks in the system.
@@ -4101,7 +4101,7 @@ public class BlockManager implements BlockStatsMXBean {
&& !Thread.currentThread().isInterrupted()
&& iter.hasNext()) {
int limit = processed + numBlocksPerIteration;
- namesystem.writeLockInterruptibly();
+ namesystem.writeLockInterruptibly(FSNamesystemLockMode.GLOBAL);
try {
while (iter.hasNext() && processed < limit) {
BlockInfo blk = iter.next();
@@ -4111,7 +4111,7 @@ public class BlockManager implements BlockStatsMXBean {
blk, r);
}
} finally {
- namesystem.writeUnlock("processMisReplicatedBlocks");
+ namesystem.writeUnlock(FSNamesystemLockMode.GLOBAL, "processMisReplicatedBlocks");
}
}
} catch (InterruptedException ex) {
@@ -5648,7 +5648,7 @@ public class BlockManager implements BlockStatsMXBean {
// batch as many operations in the write lock until the queue
// runs dry, or the max lock hold is reached.
int processed = 0;
- namesystem.writeLock();
+ namesystem.writeLock(FSNamesystemLockMode.GLOBAL);
metrics.setBlockOpsQueued(queue.size() + 1);
try {
long start = Time.monotonicNow();
@@ -5661,7 +5661,7 @@ public class BlockManager implements BlockStatsMXBean {
action = queue.poll();
} while (action != null);
} finally {
- namesystem.writeUnlock("processQueue");
+ namesystem.writeUnlock(FSNamesystemLockMode.GLOBAL, "processQueue");
metrics.addBlockOpsBatched(processed - 1);
}
} catch (InterruptedException e) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
index 7bde21dcb69e..fc1ff98a400e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.Storage.StorageState;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.apache.hadoop.util.Lists;
import org.apache.hadoop.util.StringUtils;
@@ -218,11 +219,11 @@ public class BackupImage extends FSImage {
}
lastAppliedTxId = logLoader.getLastAppliedTxId();
- getNamesystem().writeLock();
+ getNamesystem().writeLock(FSNamesystemLockMode.FS);
try {
getNamesystem().dir.updateCountForQuota();
} finally {
- getNamesystem().writeUnlock("applyEdits");
+ getNamesystem().writeUnlock(FSNamesystemLockMode.FS, "applyEdits");
}
} finally {
backupInputStream.clear();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 6237862ed4bb..8fe05a9463af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -430,7 +430,7 @@ final class FSDirErasureCodingOp {
*/
static ErasureCodingPolicyInfo[] getErasureCodingPolicies(
final FSNamesystem fsn) throws IOException {
- assert fsn.hasReadLock();
+ assert fsn.hasReadLock(FSNamesystemLockMode.FS);
return fsn.getErasureCodingPolicyManager().getPolicies();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
index e956f9ca7116..8c7007e42394 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
@@ -176,7 +176,7 @@ final class FSDirTruncateOp {
final long newLength, final long mtime, final Block truncateBlock)
throws UnresolvedLinkException, QuotaExceededException,
SnapshotAccessControlException, IOException {
- assert fsn.hasWriteLock();
+ assert fsn.hasWriteLock(FSNamesystemLockMode.GLOBAL);
FSDirectory fsd = fsn.getFSDirectory();
INodeFile file = iip.getLastINode().asFile();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index ddcf23ce55f7..1f1fe25f15f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -703,12 +703,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void setImageLoaded() {
if(imageLoaded) return;
- writeLock();
+ writeLock(FSNamesystemLockMode.GLOBAL);
try {
setImageLoaded(true);
dir.markNameCacheInitialized();
} finally {
- writeUnlock("setImageLoaded");
+ writeUnlock(FSNamesystemLockMode.GLOBAL, "setImageLoaded");
}
}
@@ -1283,7 +1283,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
startOpt = StartupOption.REGULAR;
}
boolean success = false;
- writeLock();
+ writeLock(FSNamesystemLockMode.GLOBAL);
try {
// We shouldn't be calling saveNamespace if we've come up in standby state.
MetaRecoveryContext recovery = startOpt.createRecoveryContext();
@@ -1315,7 +1315,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (!success) {
fsImage.close();
}
- writeUnlock("loadFSImage", true);
+ writeUnlock(FSNamesystemLockMode.GLOBAL, "loadFSImage", true);
}
imageLoadComplete();
}
@@ -1386,7 +1386,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* Stop services common to both active and standby states
*/
void stopCommonServices() {
- writeLock();
+ writeLock(FSNamesystemLockMode.GLOBAL);
if (inodeAttributeProvider != null) {
dir.setINodeAttributeProvider(null);
inodeAttributeProvider.stop();
@@ -1394,7 +1394,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
try {
if (blockManager != null) blockManager.close();
} finally {
- writeUnlock("stopCommonServices");
+ writeUnlock(FSNamesystemLockMode.GLOBAL, "stopCommonServices");
}
RetryCache.clear(retryCache);
}
@@ -1523,7 +1523,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
void stopActiveServices() {
LOG.info("Stopping services started for active state");
- writeLock();
+ writeLock(FSNamesystemLockMode.GLOBAL);
try {
if (blockManager != null) {
blockManager.stopReconstructionInitializer();
@@ -1571,7 +1571,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
blockManager.setInitializedReplQueues(false);
}
} finally {
- writeUnlock("stopActiveServices");
+ writeUnlock(FSNamesystemLockMode.GLOBAL, "stopActiveServices");
}
}
@@ -4648,7 +4648,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
List<BlockCollection> filesToDelete = new ArrayList<>();
boolean changed = false;
- writeLock();
+ writeLock(FSNamesystemLockMode.GLOBAL);
try {
final Iterator<BlockInfo> it =
blockManager.getCorruptReplicaBlockIterator();
@@ -4679,7 +4679,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
}
} finally {
- writeUnlock("clearCorruptLazyPersistFiles");
+ writeUnlock(FSNamesystemLockMode.GLOBAL, "clearCorruptLazyPersistFiles");
}
if (changed) {
getEditLog().logSync();
@@ -4976,12 +4976,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
int getNumberOfDatanodes(DatanodeReportType type) {
- readLock();
+ readLock(FSNamesystemLockMode.BM);
try {
return getBlockManager().getDatanodeManager().getDatanodeListForReport(
type).size();
} finally {
- readUnlock("getNumberOfDatanodes");
+ readUnlock(FSNamesystemLockMode.BM, "getNumberOfDatanodes");
}
}
@@ -5210,12 +5210,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
public long getCompleteBlocksTotal() {
// Calculate number of blocks under construction
long numUCBlocks = 0;
- readLock();
+ readLock(FSNamesystemLockMode.GLOBAL);
try {
numUCBlocks = leaseManager.getNumUnderConstructionBlocks();
return getBlocksTotal() - numUCBlocks;
} finally {
- readUnlock("getCompleteBlocksTotal");
+ readUnlock(FSNamesystemLockMode.GLOBAL, "getCompleteBlocksTotal");
}
}
@@ -7643,7 +7643,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (upgradeInfo.createdRollbackImages()) {
return new RollingUpgradeInfo.Bean(upgradeInfo);
}
- readLock();
+ readLock(FSNamesystemLockMode.FS);
try {
// check again after acquiring the read lock.
upgradeInfo = getRollingUpgradeInfo();
@@ -7657,7 +7657,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
} catch (IOException ioe) {
LOG.warn("Encountered exception setting Rollback Image", ioe);
} finally {
- readUnlock("getRollingUpgradeStatus");
+ readUnlock(FSNamesystemLockMode.FS, "getRollingUpgradeStatus");
}
return new RollingUpgradeInfo.Bean(upgradeInfo);
}
@@ -7740,7 +7740,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final String operationName = "finalizeRollingUpgrade";
checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.WRITE);
- writeLock();
+ writeLock(FSNamesystemLockMode.FS);
try {
checkOperation(OperationCategory.WRITE);
if (!isRollingUpgrade()) {
@@ -7758,7 +7758,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
getFSImage().renameCheckpoint(NameNodeFile.IMAGE_ROLLBACK,
NameNodeFile.IMAGE);
} finally {
- writeUnlock(operationName, getLockReportInfoSupplier(null));
+ writeUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(null));
}
if (!haEnabled) {
@@ -8733,7 +8733,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
@Override
public void removeXattr(long id, String xattrName) throws IOException {
- writeLock();
+ writeLock(FSNamesystemLockMode.FS);
try {
final INode inode = dir.getInode(id);
if (inode == null) {
@@ -8749,7 +8749,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
FSDirSatisfyStoragePolicyOp.removeSPSXattr(dir, inode, spsXAttr);
}
} finally {
- writeUnlock("removeXAttr");
+ writeUnlock(FSNamesystemLockMode.FS, "removeXAttr");
}
getEditLog().logSync();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index 6ba4bf00b9f8..d978ae401255 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -25,6 +25,7 @@ import java.util.Optional;
import java.util.Stack;
import java.util.function.LongFunction;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.util.Time;
@@ -49,7 +50,7 @@ import org.apache.hadoop.security.UserGroupInformation;
* The state of this class need not be synchronized as it has data structures that
* are read-only.
*
- * Some of the helper methods are guarded by {@link FSNamesystem#readLock()}.
+ * Some of the helper methods are guarded by {@link FSNamesystem#readLock(FSNamesystemLockMode)}.
*/
public class FSPermissionChecker implements AccessControlEnforcer {
static final Logger LOG = LoggerFactory.getLogger(UserGroupInformation.class);
@@ -339,7 +340,7 @@ public class FSPermissionChecker implements AccessControlEnforcer {
* @param ignoreEmptyDir Ignore permission checking for empty directory?
* @throws AccessControlException
*
- * Guarded by {@link FSNamesystem#readLock()}
+ * Guarded by {@link FSNamesystem#readLock(FSNamesystemLockMode)}
* Caller of this method must hold that lock.
*/
void checkPermission(INodesInPath inodesInPath, boolean doCheckOwner,
@@ -554,7 +555,7 @@ public class FSPermissionChecker implements AccessControlEnforcer {
return inodeAttrs;
}
- /** Guarded by {@link FSNamesystem#readLock()} */
+ /** Guarded by {@link FSNamesystem#readLock(FSNamesystemLockMode)}. */
private void checkOwner(INodeAttributes[] inodes, byte[][] components, int i)
throws AccessControlException {
if (getUser().equals(inodes[i].getUserName())) {
@@ -565,7 +566,7 @@ public class FSPermissionChecker implements AccessControlEnforcer {
" is not the owner of inode=" + getPath(components, 0, i));
}
- /** Guarded by {@link FSNamesystem#readLock()}
+ /** Guarded by {@link FSNamesystem#readLock(FSNamesystemLockMode)}.
* @throws AccessControlException
* @throws ParentNotDirectoryException
* @throws UnresolvedPathException
@@ -579,7 +580,7 @@ public class FSPermissionChecker implements AccessControlEnforcer {
}
}
- /** Guarded by {@link FSNamesystem#readLock()} */
+ /** Guarded by {@link FSNamesystem#readLock(FSNamesystemLockMode)}. */
private void checkSubAccess(byte[][] components, int pathIdx,
INode inode, int snapshotId, FsAction access, boolean ignoreEmptyDir)
throws AccessControlException {
@@ -653,7 +654,7 @@ public class FSPermissionChecker implements AccessControlEnforcer {
}
}
- /** Guarded by {@link FSNamesystem#readLock()} */
+ /** Guarded by {@link FSNamesystem#readLock(FSNamesystemLockMode)}. */
private void check(INodeAttributes[] inodes, byte[][] components, int i,
FsAction access) throws AccessControlException {
INodeAttributes inode = (i >= 0) ? inodes[i] : null;
@@ -767,7 +768,7 @@ public class FSPermissionChecker implements AccessControlEnforcer {
return !foundMatch && mode.getOtherAction().implies(access);
}
- /** Guarded by {@link FSNamesystem#readLock()} */
+ /** Guarded by {@link FSNamesystem#readLock(FSNamesystemLockMode)}. */
private void checkStickyBit(INodeAttributes[] inodes, byte[][] components,
int index) throws AccessControlException {
INodeAttributes parent = inodes[index];
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java
index c1d26f40c082..12b04d221a7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java
@@ -28,6 +28,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
import org.apache.hadoop.util.Timer;
@@ -127,7 +128,7 @@ public abstract class FSTreeTraverser {
List<byte[]> startAfters, final TraverseInfo traverseInfo)
throws IOException, InterruptedException {
assert dir.hasReadLock();
- assert dir.getFSNamesystem().hasReadLock();
+ assert dir.getFSNamesystem().hasReadLock(FSNamesystemLockMode.FS);
long lockStartTime = timer.monotonicNow();
Preconditions.checkNotNull(curr, "Current inode can't be null");
checkINodeReady(startId);
@@ -261,13 +262,13 @@ public abstract class FSTreeTraverser {
}
protected void readLock() {
- dir.getFSNamesystem().readLock();
+ dir.getFSNamesystem().readLock(FSNamesystemLockMode.FS);
dir.readLock();
}
protected void readUnlock() {
dir.readUnlock();
- dir.getFSNamesystem().readUnlock("FSTreeTraverser");
+ dir.getFSNamesystem().readUnlock(FSNamesystemLockMode.FS, "FSTreeTraverser");
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
index 275be9050511..4752f11e09ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
import org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics;
@@ -272,13 +273,13 @@ public class FsImageValidation {
final FSImageFormat.LoaderDelegator loader
= FSImageFormat.newLoader(conf, namesystem);
- namesystem.writeLock();
+ namesystem.writeLock(FSNamesystemLockMode.GLOBAL);
namesystem.getFSDirectory().writeLock();
try {
loader.load(fsImageFile, false);
} finally {
namesystem.getFSDirectory().writeUnlock();
- namesystem.writeUnlock("loadImage");
+ namesystem.writeUnlock(FSNamesystemLockMode.GLOBAL, "loadImage");
}
}
t.cancel();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index c2e01ffe266f..df1dc3549856 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -136,8 +136,8 @@ public class LeaseManager {
* calling this method.
*/
synchronized long getNumUnderConstructionBlocks() {
- assert this.fsnamesystem.hasReadLock() : "The FSNamesystem read lock wasn't"
- + "acquired before counting under construction blocks";
+ assert this.fsnamesystem.hasReadLock(FSNamesystemLockMode.GLOBAL) :
+ "The FSNamesystem read lock wasn't acquired before counting under construction blocks";
long numUCBlocks = 0;
for (Long id : getINodeIdWithLeases()) {
INode inode = fsnamesystem.getFSDirectory().getInode(id);
@@ -208,7 +208,7 @@ public class LeaseManager {
*/
public Set<INodesInPath> getINodeWithLeases(final INodeDirectory
ancestorDir) throws IOException {
- assert fsnamesystem.hasReadLock();
+ assert fsnamesystem.hasReadLock(FSNamesystemLockMode.FS);
final long startTimeMs = Time.monotonicNow();
Set<INodesInPath> iipSet = new HashSet<>();
final INode[] inodes = getINodesWithLease();
@@ -543,13 +543,13 @@ public class LeaseManager {
continue;
}
- fsnamesystem.writeLockInterruptibly();
+ fsnamesystem.writeLockInterruptibly(FSNamesystemLockMode.GLOBAL);
try {
if (!fsnamesystem.isInSafeMode()) {
needSync = checkLeases(candidates);
}
} finally {
- fsnamesystem.writeUnlock("leaseManager");
+ fsnamesystem.writeUnlock(FSNamesystemLockMode.GLOBAL, "leaseManager");
// lease reassignments should to be sync'ed.
if (needSync) {
fsnamesystem.getEditLog().logSync();
@@ -574,7 +574,7 @@ public class LeaseManager {
private synchronized boolean checkLeases(Collection<Lease> leasesToCheck) {
boolean needSync = false;
- assert fsnamesystem.hasWriteLock();
+ assert fsnamesystem.hasWriteLock(FSNamesystemLockMode.GLOBAL);
long start = monotonicNow();
for (Lease leaseToCheck : leasesToCheck) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index a48cfdbe5957..c42d75678283 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
import org.apache.hadoop.util.Preconditions;
@@ -2238,14 +2239,14 @@ public class NameNode extends ReconfigurableBase implements
@Override
public void writeLock() {
- namesystem.writeLock();
+ namesystem.writeLock(FSNamesystemLockMode.GLOBAL);
namesystem.lockRetryCache();
}
@Override
public void writeUnlock() {
namesystem.unlockRetryCache();
- namesystem.writeUnlock("HAState");
+ namesystem.writeUnlock(FSNamesystemLockMode.GLOBAL, "HAState");
}
/** Check if an operation of given category is allowed */
@@ -2396,7 +2397,7 @@ public class NameNode extends ReconfigurableBase implements
final String property) throws ReconfigurationException {
BlockManager bm = namesystem.getBlockManager();
int newSetting;
- namesystem.writeLock();
+ namesystem.writeLock(FSNamesystemLockMode.BM);
try {
if (property.equals(DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY)) {
bm.setMaxReplicationStreams(
@@ -2434,7 +2435,7 @@ public class NameNode extends ReconfigurableBase implements
throw new ReconfigurationException(property, newVal, getConf().get(
property), e);
} finally {
- namesystem.writeUnlock("reconfReplicationParameters");
+ namesystem.writeUnlock(FSNamesystemLockMode.BM, "reconfReplicationParameters");
}
}
@@ -2454,7 +2455,7 @@ public class NameNode extends ReconfigurableBase implements
private String reconfHeartbeatInterval(final DatanodeManager datanodeManager,
final String property, final String newVal)
throws ReconfigurationException {
- namesystem.writeLock();
+ namesystem.writeLock(FSNamesystemLockMode.BM);
try {
if (newVal == null) {
// set to default
@@ -2471,7 +2472,7 @@ public class NameNode extends ReconfigurableBase implements
throw new ReconfigurationException(property, newVal, getConf().get(
property), nfe);
} finally {
- namesystem.writeUnlock("reconfHeartbeatInterval");
+ namesystem.writeUnlock(FSNamesystemLockMode.BM, "reconfHeartbeatInterval");
LOG.info("RECONFIGURE* changed heartbeatInterval to "
+ datanodeManager.getHeartbeatInterval());
}
@@ -2480,7 +2481,7 @@ public class NameNode extends ReconfigurableBase implements
private String reconfHeartbeatRecheckInterval(
final DatanodeManager datanodeManager, final String property,
final String newVal) throws ReconfigurationException {
- namesystem.writeLock();
+ namesystem.writeLock(FSNamesystemLockMode.BM);
try {
if (newVal == null) {
// set to default
@@ -2495,7 +2496,7 @@ public class NameNode extends ReconfigurableBase implements
throw new ReconfigurationException(property, newVal, getConf().get(
property), nfe);
} finally {
- namesystem.writeUnlock("reconfHeartbeatRecheckInterval");
+ namesystem.writeUnlock(FSNamesystemLockMode.BM, "reconfHeartbeatRecheckInterval");
LOG.info("RECONFIGURE* changed heartbeatRecheckInterval to "
+ datanodeManager.getHeartbeatRecheckInterval());
}
@@ -2620,7 +2621,7 @@ public class NameNode extends ReconfigurableBase implements
String reconfigureSlowNodesParameters(final DatanodeManager datanodeManager,
final String property, final String newVal) throws ReconfigurationException {
BlockManager bm = namesystem.getBlockManager();
- namesystem.writeLock();
+ namesystem.writeLock(FSNamesystemLockMode.BM);
String result;
try {
switch (property) {
@@ -2697,13 +2698,13 @@ public class NameNode extends ReconfigurableBase implements
throw new ReconfigurationException(property, newVal, getConf().get(
property), e);
} finally {
- namesystem.writeUnlock("reconfigureSlowNodesParameters");
+ namesystem.writeUnlock(FSNamesystemLockMode.BM, "reconfigureSlowNodesParameters");
}
}
private String reconfigureBlockInvalidateLimit(final DatanodeManager datanodeManager,
final String property, final String newVal) throws ReconfigurationException {
- namesystem.writeLock();
+ namesystem.writeLock(FSNamesystemLockMode.BM);
try {
if (newVal == null) {
datanodeManager.setBlockInvalidateLimit(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT);
@@ -2717,7 +2718,7 @@ public class NameNode extends ReconfigurableBase implements
} catch (NumberFormatException e) {
throw new ReconfigurationException(property, newVal, getConf().get(property), e);
} finally {
- namesystem.writeUnlock("reconfigureBlockInvalidateLimit");
+ namesystem.writeUnlock(FSNamesystemLockMode.BM, "reconfigureBlockInvalidateLimit");
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index bc6df0141c64..5c2ddfbdd0c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -38,6 +38,7 @@ import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockUnderConstructionFeature;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.apache.hadoop.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -289,7 +290,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
return;
}
- namenode.getNamesystem().readLock();
+ // TODO: Just hold the BM read lock.
+ namenode.getNamesystem().readLock(FSNamesystemLockMode.GLOBAL);
try {
//get blockInfo
Block block = new Block(Block.getBlockId(blockId));
@@ -353,7 +355,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
out.print("\n\n" + errMsg);
LOG.warn("Error in looking up block", e);
} finally {
- namenode.getNamesystem().readUnlock("fsck");
+ namenode.getNamesystem().readUnlock(FSNamesystemLockMode.GLOBAL, "fsck");
}
}
@@ -585,7 +587,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
final String operationName = "fsckGetBlockLocations";
FSPermissionChecker.setOperationType(operationName);
FSPermissionChecker pc = fsn.getPermissionChecker();
- fsn.readLock();
+ fsn.readLock(FSNamesystemLockMode.GLOBAL);
try {
blocks = FSDirStatAndListingOp.getBlockLocations(
fsn.getFSDirectory(), pc,
@@ -594,7 +596,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
} catch (FileNotFoundException fnfe) {
blocks = null;
} finally {
- fsn.readUnlock(operationName);
+ fsn.readUnlock(FSNamesystemLockMode.GLOBAL, operationName);
}
return blocks;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
index e95200b35aaa..910b9568eb1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
@@ -37,6 +37,7 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
+import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -1094,11 +1095,11 @@ public class SecondaryNameNode implements Runnable,
sig.mostRecentCheckpointTxId + " even though it should have " +
"just been downloaded");
}
- dstNamesystem.writeLock();
+ dstNamesystem.writeLock(FSNamesystemLockMode.GLOBAL);
try {
dstImage.reloadFromImageFile(file, dstNamesystem);
} finally {
- dstNamesystem.writeUnlock("reloadFromImageFile");
+ dstNamesystem.writeUnlock(FSNamesystemLockMode.GLOBAL, "reloadFromImageFile");
}
dstNamesystem.imageLoadComplete();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
index 6be34537a050..5e6ab565a3eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
@@ -469,6 +469,8 @@ public class TestLeaseManager {
when(fsn.hasWriteLock()).thenReturn(true);
when(fsn.hasReadLock(FSNamesystemLockMode.FS)).thenReturn(true);
when(fsn.hasWriteLock(FSNamesystemLockMode.FS)).thenReturn(true);
+ when(fsn.hasReadLock(FSNamesystemLockMode.GLOBAL)).thenReturn(true);
+ when(fsn.hasWriteLock(FSNamesystemLockMode.GLOBAL)).thenReturn(true);
when(fsn.getFSDirectory()).thenReturn(dir);
when(fsn.getMaxLockHoldToReleaseLeaseMs()).thenReturn(maxLockHoldToReleaseLeaseMs);
return fsn;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org