[hadoop] branch branch-3.2 updated: HDFS-14806. Bootstrap standby may fail if with in-progress tailing. Contributed by Chen Liang and Erik Krogen.

[sh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

shv pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
     new c9ed0c3  HDFS-14806. Bootstrap standby may fail if with in-progress tailing. Contributed by Chen Liang and Erik Krogen.
c9ed0c3 is described below

commit c9ed0c3a3738e75d28c5a427070a85f87bbb8624
Author: Chen Liang <cl...@apache.org>
AuthorDate: Wed Nov 6 09:30:04 2019 -0800

    HDFS-14806. Bootstrap standby may fail if with in-progress tailing. Contributed by Chen Liang and Erik Krogen.
---
 .../hdfs/server/namenode/ha/BootstrapStandby.java  |  4 ++
 .../TestBootstrapStandbyWithInProgressTailing.java | 46 ++++++++++++++++++++++
 .../namenode/ha/TestBootstrapStandbyWithQJM.java   |  9 ++++-
 3 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
index aed90a5..0751e4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
@@ -31,6 +31,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -101,6 +102,9 @@ public class BootstrapStandby implements Tool, Configurable {
   @Override
   public int run(String[] args) throws Exception {
     parseArgs(args);
+    // Disable using the RPC tailing mechanism for bootstrapping the standby
+    // since it is less efficient in this case; see HDFS-14806
+    conf.setBoolean(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, false);
     parseConfAndFindOtherNN();
     NameNode.checkAllowFormat(conf);
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithInProgressTailing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithInProgressTailing.java
new file mode 100644
index 0000000..a18e8ac
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithInProgressTailing.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode.ha;
+
+import org.apache.hadoop.conf.Configuration;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY;
+
+
+/**
+ * Test in progress tailing with small txn id per call.
+ *
+ * The number of edits that needs to be tailed during
+ * bootstrapStandby can be large, but the number of edits
+ * that can be tailed using RPC call can be limited
+ * (configured by dfs.ha.tail-edits.qjm.rpc.max-txns).
+ * This is to test that even with small number of configured
+ * txnid, bootstrapStandby can still work. See HDFS-14806.
+ */
+public class TestBootstrapStandbyWithInProgressTailing
+    extends TestBootstrapStandbyWithQJM {
+  @Override
+  public Configuration createConfig() {
+    Configuration conf = super.createConfig();
+
+    conf.setBoolean(DFS_HA_TAILEDITS_INPROGRESS_KEY, true);
+    conf.setInt("dfs.ha.tail-edits.qjm.rpc.max-txns", 1);
+
+    return conf;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java
index 6332451..7739bc7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java
@@ -54,14 +54,19 @@ public class TestBootstrapStandbyWithQJM {
   private MiniJournalCluster jCluster;
   private int nnCount = 3;
 
-  @Before
-  public void setup() throws Exception {
+  protected Configuration createConfig() {
     Configuration conf = new Configuration();
     // Turn off IPC client caching, so that the suite can handle
     // the restart of the daemons between test cases.
     conf.setInt(
         CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
         0);
+    return conf;
+  }
+
+  @Before
+  public void setup() throws Exception {
+    Configuration conf = createConfig();
 
     MiniQJMHACluster miniQjmHaCluster =
         new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org