You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cl...@apache.org on 2019/03/29 02:08:24 UTC
[hadoop] branch branch-2 updated: HDFS-14392. Backport HDFS-9787 to
branch-2. Contributed by Chao Sun.
This is an automated email from the ASF dual-hosted git repository.
cliang pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 812e6d5 HDFS-14392. Backport HDFS-9787 to branch-2. Contributed by Chao Sun.
812e6d5 is described below
commit 812e6d5c81414a5b36683976c8852deb1e6fb4cd
Author: Chen Liang <cl...@apache.org>
AuthorDate: Thu Mar 28 19:07:58 2019 -0700
HDFS-14392. Backport HDFS-9787 to branch-2. Contributed by Chao Sun.
---
.../server/namenode/ha/StandbyCheckpointer.java | 7 +++-
.../server/namenode/ha/TestStandbyCheckpoints.java | 42 ++++++++++++++++++++++
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
index b2f2cac..753447b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
@@ -65,6 +65,7 @@ public class StandbyCheckpointer {
private final Configuration conf;
private final FSNamesystem namesystem;
private long lastCheckpointTime;
+ private long lastUploadTime;
private final CheckpointerThread thread;
private final ThreadFactory uploadThreadFactory;
private List<URL> activeNNAddresses;
@@ -257,6 +258,7 @@ public class StandbyCheckpointer {
break;
}
}
+ lastUploadTime = monotonicNow();
// we are primary if we successfully updated the ANN
this.isPrimaryCheckPointer = success;
@@ -367,6 +369,7 @@ public class StandbyCheckpointer {
// Reset checkpoint time so that we don't always checkpoint
// on startup.
lastCheckpointTime = monotonicNow();
+ lastUploadTime = monotonicNow();
while (shouldRun) {
boolean needRollbackCheckpoint = namesystem.isNeedRollbackFsImage();
if (!needRollbackCheckpoint) {
@@ -419,7 +422,9 @@ public class StandbyCheckpointer {
// on all nodes, we build the checkpoint. However, we only ship the checkpoint if have a
// rollback request, are the checkpointer, are outside the quiet period.
- boolean sendRequest = isPrimaryCheckPointer || secsSinceLast >= checkpointConf.getQuietPeriod();
+ final long secsSinceLastUpload = (now - lastUploadTime) / 1000;
+ boolean sendRequest = isPrimaryCheckPointer
+ || secsSinceLastUpload >= checkpointConf.getQuietPeriod();
doCheckpoint(sendRequest);
// reset needRollbackCheckpoint to false only when we finish a ckpt
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
index af75e42..75b2412 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
@@ -527,6 +527,48 @@ public class TestStandbyCheckpoints {
HATestUtil.waitForCheckpoint(cluster, 0, ImmutableList.of(12));
}
+ /**
+ * Test for the case standby NNs can upload FSImage to ANN after
+ * become non-primary standby NN. HDFS-9787
+ */
+ @Test(timeout=300000)
+ public void testNonPrimarySBNUploadFSImage() throws Exception {
+ // Shutdown all standby NNs.
+ for (int i = 1; i < NUM_NNS; i++) {
+ cluster.shutdownNameNode(i);
+
+ // Checkpoint as fast as we can, in a tight loop.
+ cluster.getConfiguration(i).setInt(
+ DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1);
+ }
+
+ doEdits(0, 10);
+ cluster.transitionToStandby(0);
+
+ // Standby NNs do checkpoint without active NN available.
+ for (int i = 1; i < NUM_NNS; i++) {
+ cluster.restartNameNode(i, false);
+ }
+ cluster.waitClusterUp();
+
+ for (int i = 0; i < NUM_NNS; i++) {
+ // Once the standby catches up, it should do a checkpoint
+ // and save to local directories.
+ HATestUtil.waitForCheckpoint(cluster, 1, ImmutableList.of(12));
+ }
+
+ cluster.transitionToActive(0);
+
+ // Wait for 2 seconds to expire last upload time.
+ Thread.sleep(2000);
+
+ doEdits(11, 20);
+ nns[0].getRpcServer().rollEditLog();
+
+ // One of standby NNs should also upload it back to the active.
+ HATestUtil.waitForCheckpoint(cluster, 0, ImmutableList.of(23));
+ }
+
private void doEdits(int start, int stop) throws IOException {
for (int i = start; i < stop; i++) {
Path p = new Path("/test" + i);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org