You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2019/11/07 00:22:34 UTC
[hadoop] branch branch-3.1 updated: HDFS-14806. Bootstrap standby
may fail if with in-progress tailing. Contributed by Chen Liang and Erik
Krogen.
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 76ca864 HDFS-14806. Bootstrap standby may fail if with in-progress tailing. Contributed by Chen Liang and Erik Krogen.
76ca864 is described below
commit 76ca86440672a7067ee582ae851c1686530c7c72
Author: Chen Liang <cl...@apache.org>
AuthorDate: Wed Nov 6 09:30:04 2019 -0800
HDFS-14806. Bootstrap standby may fail if with in-progress tailing. Contributed by Chen Liang and Erik Krogen.
---
.../hdfs/server/namenode/ha/BootstrapStandby.java | 4 ++
.../TestBootstrapStandbyWithInProgressTailing.java | 46 ++++++++++++++++++++++
.../namenode/ha/TestBootstrapStandbyWithQJM.java | 9 ++++-
3 files changed, 57 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
index 4d6716f..e69baf3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HAUtil;
@@ -100,6 +101,9 @@ public class BootstrapStandby implements Tool, Configurable {
@Override
public int run(String[] args) throws Exception {
parseArgs(args);
+ // Disable using the RPC tailing mechanism for bootstrapping the standby
+ // since it is less efficient in this case; see HDFS-14806
+ conf.setBoolean(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, false);
parseConfAndFindOtherNN();
NameNode.checkAllowFormat(conf);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithInProgressTailing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithInProgressTailing.java
new file mode 100644
index 0000000..a18e8ac
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithInProgressTailing.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode.ha;
+
+import org.apache.hadoop.conf.Configuration;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY;
+
+
+/**
+ * Test in progress tailing with small txn id per call.
+ *
+ * The number of edits that needs to be tailed during
+ * bootstrapStandby can be large, but the number of edits
+ * that can be tailed using RPC call can be limited
+ * (configured by dfs.ha.tail-edits.qjm.rpc.max-txns).
+ * This is to test that even with small number of configured
+ * txnid, bootstrapStandby can still work. See HDFS-14806.
+ */
+public class TestBootstrapStandbyWithInProgressTailing
+ extends TestBootstrapStandbyWithQJM {
+ @Override
+ public Configuration createConfig() {
+ Configuration conf = super.createConfig();
+
+ conf.setBoolean(DFS_HA_TAILEDITS_INPROGRESS_KEY, true);
+ conf.setInt("dfs.ha.tail-edits.qjm.rpc.max-txns", 1);
+
+ return conf;
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java
index 10194ba..aad01f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java
@@ -54,14 +54,19 @@ public class TestBootstrapStandbyWithQJM {
private MiniJournalCluster jCluster;
private int nnCount = 3;
- @Before
- public void setup() throws Exception {
+ protected Configuration createConfig() {
Configuration conf = new Configuration();
// Turn off IPC client caching, so that the suite can handle
// the restart of the daemons between test cases.
conf.setInt(
CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
0);
+ return conf;
+ }
+
+ @Before
+ public void setup() throws Exception {
+ Configuration conf = createConfig();
MiniQJMHACluster miniQjmHaCluster =
new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org