You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2017/06/09 03:04:37 UTC
[22/29] hadoop git commit: HDFS-11945. Internal lease recovery may
not be retried for a long time. Contributed by Kihwal Lee
HDFS-11945. Internal lease recovery may not be retried for a long time. Contributed by Kihwal Lee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a33c9d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a33c9d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a33c9d5
Branch: refs/heads/YARN-5734
Commit: 1a33c9d58927186c2f219a5ecb5f1573801823ad
Parents: 164c222
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Jun 8 14:36:47 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Jun 8 14:36:47 2017 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/LeaseManager.java | 27 +++++++++++++++++---
.../apache/hadoop/hdfs/TestLeaseRecovery2.java | 12 +++++----
.../hdfs/server/namenode/TestFileTruncate.java | 2 +-
.../hdfs/server/namenode/TestLeaseManager.java | 14 ++++++++++
4 files changed, 46 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a33c9d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index dcdb958..8695d63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -79,6 +79,8 @@ public class LeaseManager {
private long hardLimit = HdfsConstants.LEASE_HARDLIMIT_PERIOD;
static final int INODE_FILTER_WORKER_COUNT_MAX = 4;
static final int INODE_FILTER_WORKER_TASK_MIN = 512;
+ private long lastHolderUpdateTime;
+ private String internalLeaseHolder;
// Used for handling lock-leases
// Mapping: leaseHolder -> Lease
@@ -97,7 +99,26 @@ public class LeaseManager {
private Daemon lmthread;
private volatile boolean shouldRunMonitor;
- LeaseManager(FSNamesystem fsnamesystem) {this.fsnamesystem = fsnamesystem;}
+ LeaseManager(FSNamesystem fsnamesystem) {
+ this.fsnamesystem = fsnamesystem;
+ updateInternalLeaseHolder();
+ }
+
+ // Update the internal lease holder with the current time stamp.
+ private void updateInternalLeaseHolder() {
+ this.lastHolderUpdateTime = Time.monotonicNow();
+ this.internalLeaseHolder = HdfsServerConstants.NAMENODE_LEASE_HOLDER +
+ "-" + Time.formatTime(Time.now());
+ }
+
+ // Get the current internal lease holder name.
+ String getInternalLeaseHolder() {
+ long elapsed = Time.monotonicNow() - lastHolderUpdateTime;
+ if (elapsed > hardLimit) {
+ updateInternalLeaseHolder();
+ }
+ return internalLeaseHolder;
+ }
Lease getLease(String holder) {
return leases.get(holder);
@@ -467,6 +488,7 @@ public class LeaseManager {
Long[] leaseINodeIds = files.toArray(new Long[files.size()]);
FSDirectory fsd = fsnamesystem.getFSDirectory();
String p = null;
+ String newHolder = getInternalLeaseHolder();
for(Long id : leaseINodeIds) {
try {
INodesInPath iip = INodesInPath.fromINode(fsd.getInode(id));
@@ -478,8 +500,7 @@ public class LeaseManager {
boolean completed = false;
try {
completed = fsnamesystem.internalReleaseLease(
- leaseToCheck, p, iip,
- HdfsServerConstants.NAMENODE_LEASE_HOLDER);
+ leaseToCheck, p, iip, newHolder);
} catch (IOException e) {
LOG.warn("Cannot release the path " + p + " in the lease "
+ leaseToCheck + ". It will be retried.", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a33c9d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
index e8cd476..a807bbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
@@ -465,7 +465,8 @@ public class TestLeaseRecovery2 {
cluster.getNameNode(), fileStr);
assertFalse("original lease holder should not be the NN",
- originalLeaseHolder.equals(HdfsServerConstants.NAMENODE_LEASE_HOLDER));
+ originalLeaseHolder.startsWith(
+ HdfsServerConstants.NAMENODE_LEASE_HOLDER));
// hflush file
AppendTestUtil.LOG.info("hflush");
@@ -501,8 +502,9 @@ public class TestLeaseRecovery2 {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
- return HdfsServerConstants.NAMENODE_LEASE_HOLDER.equals(
- NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), path));
+ String holder =
+ NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), path);
+ return holder.startsWith(HdfsServerConstants.NAMENODE_LEASE_HOLDER);
}
}, (int)SHORT_LEASE_PERIOD, (int)SHORT_LEASE_PERIOD * 10);
@@ -563,8 +565,8 @@ public class TestLeaseRecovery2 {
if (size == 0) {
assertEquals("lease holder should null, file is closed", null, holder);
} else {
- assertEquals("lease holder should now be the NN",
- HdfsServerConstants.NAMENODE_LEASE_HOLDER, holder);
+ assertTrue("lease holder should now be the NN",
+ holder.startsWith(HdfsServerConstants.NAMENODE_LEASE_HOLDER));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a33c9d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index 0ea587c..d4215e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -642,7 +642,7 @@ public class TestFileTruncate {
String leaseHolder =
NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(),
p.toUri().getPath());
- if(leaseHolder.equals(HdfsServerConstants.NAMENODE_LEASE_HOLDER)) {
+ if(leaseHolder.startsWith(HdfsServerConstants.NAMENODE_LEASE_HOLDER)) {
recoveryTriggered = true;
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a33c9d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
index 9adb071..74752f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import com.google.common.collect.Lists;
@@ -99,6 +100,19 @@ public class TestLeaseManager {
assertTrue(lm.countLease() < numLease);
}
+ /**
+ * Test whether the internal lease holder name is updated properly.
+ */
+ @Test
+ public void testInternalLeaseHolder() throws Exception {
+ LeaseManager lm = new LeaseManager(makeMockFsNameSystem());
+ // Set the hard lease limit to 500ms.
+ lm.setLeasePeriod(100L, 500L);
+ String holder = lm.getInternalLeaseHolder();
+ Thread.sleep(1000);
+ assertNotEquals(holder, lm.getInternalLeaseHolder());
+ }
+
@Test
public void testCountPath() {
LeaseManager lm = new LeaseManager(makeMockFsNameSystem());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org