You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yj...@apache.org on 2016/01/20 07:55:59 UTC
hadoop git commit: HDFS-6054. MiniQJMHACluster should not use static
port to avoid binding failure in unit test. (Yongjun Zhang)
Repository: hadoop
Updated Branches:
refs/heads/trunk 1acc509b4 -> 57d0a9430
HDFS-6054. MiniQJMHACluster should not use static port to avoid binding failure in unit test. (Yongjun Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57d0a943
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57d0a943
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57d0a943
Branch: refs/heads/trunk
Commit: 57d0a94305b22a469062c7ce85b3159fee17a28b
Parents: 1acc509
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Tue Jan 19 22:54:47 2016 -0800
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Tue Jan 19 22:54:47 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 15 +++++--
.../namenode/ha/TestFailureToReadEdits.java | 41 ++++++++++++++++----
3 files changed, 47 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57d0a943/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 327cf73..b051f2f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2599,6 +2599,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9623. Update example configuration of block state change log in
log4j.properties. (Masatake Iwasaki via aajisaka)
+ HDFS-6054. MiniQJMHACluster should not use static port to avoid binding
+ failure in unit test. (Yongjun Zhang)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57d0a943/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
index 8e838c3..0764f12 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
@@ -42,7 +42,6 @@ public class MiniQJMHACluster {
public static final String NAMESERVICE = "ns1";
private static final Random RANDOM = new Random();
- private int basePort = 10000;
public static class Builder {
private final Configuration conf;
@@ -92,9 +91,12 @@ public class MiniQJMHACluster {
private MiniQJMHACluster(Builder builder) throws IOException {
this.conf = builder.conf;
int retryCount = 0;
+ int basePort = 10000;
+
while (true) {
try {
basePort = 10000 + RANDOM.nextInt(1000) * 4;
+ LOG.info("Set MiniQJMHACluster basePort to " + basePort);
// start 3 journal nodes
journalCluster = new MiniJournalCluster.Builder(conf).format(true)
.build();
@@ -104,7 +106,7 @@ public class MiniQJMHACluster {
// start cluster with specified NameNodes
MiniDFSNNTopology topology = createDefaultTopology(builder.numNNs, basePort);
- initHAConf(journalURI, builder.conf, builder.numNNs);
+ initHAConf(journalURI, builder.conf, builder.numNNs, basePort);
// First start up the NNs just to format the namespace. The MinIDFSCluster
// has no way to just format the NameNodes without also starting them.
@@ -123,16 +125,21 @@ public class MiniQJMHACluster {
// restart the cluster
cluster.restartNameNodes();
- ++retryCount;
break;
} catch (BindException e) {
+ if (cluster != null) {
+ cluster.shutdown(true);
+ cluster = null;
+ }
+ ++retryCount;
LOG.info("MiniQJMHACluster port conflicts, retried " +
retryCount + " times");
}
}
}
- private Configuration initHAConf(URI journalURI, Configuration conf, int numNNs) {
+ private Configuration initHAConf(URI journalURI, Configuration conf,
+ int numNNs, int basePort) {
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
journalURI.toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57d0a943/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
index f82b616..5221ef9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
@@ -27,10 +27,14 @@ import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.spy;
import java.io.IOException;
+import java.net.BindException;
import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
+import java.util.Random;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -62,11 +66,14 @@ import com.google.common.collect.ImmutableList;
@RunWith(Parameterized.class)
public class TestFailureToReadEdits {
+ private static final Log LOG =
+ LogFactory.getLog(TestFailureToReadEdits.class);
private static final String TEST_DIR1 = "/test1";
private static final String TEST_DIR2 = "/test2";
private static final String TEST_DIR3 = "/test3";
-
+ private static final Random RANDOM = new Random();
+
private final TestType clusterType;
private Configuration conf;
private MiniDFSCluster cluster;
@@ -103,14 +110,32 @@ public class TestFailureToReadEdits {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
HAUtil.setAllowStandbyReads(conf, true);
-
+
if (clusterType == TestType.SHARED_DIR_HA) {
- MiniDFSNNTopology topology = MiniQJMHACluster.createDefaultTopology(10000);
- cluster = new MiniDFSCluster.Builder(conf)
- .nnTopology(topology)
- .numDataNodes(0)
- .checkExitOnShutdown(false)
- .build();
+ int basePort = 10000;
+ int retryCount = 0;
+ while (true) {
+ try {
+ basePort = 10000 + RANDOM.nextInt(1000) * 4;
+ LOG.info("Set SHARED_DIR_HA cluster's basePort to " + basePort);
+ MiniDFSNNTopology topology =
+ MiniQJMHACluster.createDefaultTopology(basePort);
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(topology)
+ .numDataNodes(0)
+ .checkExitOnShutdown(false)
+ .build();
+ break;
+ } catch (BindException e) {
+ if (cluster != null) {
+ cluster.shutdown(true);
+ cluster = null;
+ }
+ ++retryCount;
+ LOG.info("SHARED_DIR_HA: MiniQJMHACluster port conflicts, retried " +
+ retryCount + " times " + e);
+ }
+ }
} else {
Builder builder = new MiniQJMHACluster.Builder(conf);
builder.getDfsBuilder().numDataNodes(0).checkExitOnShutdown(false);