You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/02/17 10:50:33 UTC
[1/2] hadoop git commit: HDFS-7703. Support favouredNodes for the
append for new blocks ( Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 54e33baaf -> 005e1df54
HDFS-7703. Support favouredNodes for the append for new blocks ( Contributed by Vinayakumar B)
(cherry picked from commit 89a544928083501625bc69f96b530040228f0a5f)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/91a5d929
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/91a5d929
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/91a5d929
Branch: refs/heads/branch-2
Commit: 91a5d92916c6c1b0475d5794c3855b53b020d4ec
Parents: 54e33ba
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Feb 12 12:38:44 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Feb 17 15:19:03 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 53 +++++++++++++++-----
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 7 ++-
.../hadoop/hdfs/DistributedFileSystem.java | 43 ++++++++++++++++
.../namenode/TestFavoredNodesEndToEnd.java | 29 +++++++++++
5 files changed, 121 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91a5d929/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6bfa34c..a1b2053 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -324,6 +324,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7761. cleanup unnecssary code logic in LocatedBlock. (yliu)
+ HDFS-7703. Support favouredNodes for the append for new blocks
+ (vinayakumarb)
+
HDFS-7694. FSDataInputStream should support "unbuffer" (cmccabe)
HDFS-7684. The host:port settings of the daemons should be trimmed before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91a5d929/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index d27197f..3c0ec99 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1693,6 +1693,15 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
if(LOG.isDebugEnabled()) {
LOG.debug(src + ": masked=" + masked);
}
+ final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,
+ src, masked, flag, createParent, replication, blockSize, progress,
+ buffersize, dfsClientConf.createChecksum(checksumOpt),
+ getFavoredNodesStr(favoredNodes));
+ beginFileLease(result.getFileId(), result);
+ return result;
+ }
+
+ private String[] getFavoredNodesStr(InetSocketAddress[] favoredNodes) {
String[] favoredNodeStrs = null;
if (favoredNodes != null) {
favoredNodeStrs = new String[favoredNodes.length];
@@ -1702,12 +1711,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
+ favoredNodes[i].getPort();
}
}
- final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,
- src, masked, flag, createParent, replication, blockSize, progress,
- buffersize, dfsClientConf.createChecksum(checksumOpt),
- favoredNodeStrs);
- beginFileLease(result.getFileId(), result);
- return result;
+ return favoredNodeStrs;
}
/**
@@ -1725,7 +1729,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
return null;
}
- return callAppend(src, buffersize, flag, progress);
+ return callAppend(src, buffersize, flag, progress, null);
}
return null;
}
@@ -1804,7 +1808,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
/** Method to get stream returned by append call */
private DFSOutputStream callAppend(String src, int buffersize,
- EnumSet<CreateFlag> flag, Progressable progress) throws IOException {
+ EnumSet<CreateFlag> flag, Progressable progress, String[] favoredNodes)
+ throws IOException {
CreateFlag.validateForAppend(flag);
try {
LastBlockWithStatus blkWithStatus = namenode.append(src, clientName,
@@ -1812,7 +1817,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
return DFSOutputStream.newStreamForAppend(this, src,
flag.contains(CreateFlag.NEW_BLOCK),
buffersize, progress, blkWithStatus.getLastBlock(),
- blkWithStatus.getFileStatus(), dfsClientConf.createChecksum());
+ blkWithStatus.getFileStatus(), dfsClientConf.createChecksum(), favoredNodes);
} catch(RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class,
FileNotFoundException.class,
@@ -1840,14 +1845,38 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
public HdfsDataOutputStream append(final String src, final int buffersize,
EnumSet<CreateFlag> flag, final Progressable progress,
final FileSystem.Statistics statistics) throws IOException {
- final DFSOutputStream out = append(src, buffersize, flag, progress);
+ final DFSOutputStream out = append(src, buffersize, flag, null, progress);
+ return createWrappedOutputStream(out, statistics, out.getInitialLen());
+ }
+
+ /**
+ * Append to an existing HDFS file.
+ *
+ * @param src file name
+ * @param buffersize buffer size
+ * @param flag indicates whether to append data to a new block instead of the
+ * last block
+ * @param progress for reporting write-progress; null is acceptable.
+ * @param statistics file system statistics; null is acceptable.
+ * @param favoredNodes FavoredNodes for new blocks
+ * @return an output stream for writing into the file
+ * @see ClientProtocol#append(String, String, EnumSetWritable)
+ */
+ public HdfsDataOutputStream append(final String src, final int buffersize,
+ EnumSet<CreateFlag> flag, final Progressable progress,
+ final FileSystem.Statistics statistics,
+ final InetSocketAddress[] favoredNodes) throws IOException {
+ final DFSOutputStream out = append(src, buffersize, flag,
+ getFavoredNodesStr(favoredNodes), progress);
return createWrappedOutputStream(out, statistics, out.getInitialLen());
}
private DFSOutputStream append(String src, int buffersize,
- EnumSet<CreateFlag> flag, Progressable progress) throws IOException {
+ EnumSet<CreateFlag> flag, String[] favoredNodes, Progressable progress)
+ throws IOException {
checkOpen();
- final DFSOutputStream result = callAppend(src, buffersize, flag, progress);
+ final DFSOutputStream result = callAppend(src, buffersize, flag, progress,
+ favoredNodes);
beginFileLease(result.getFileId(), result);
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91a5d929/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 47b6f36..3ed957b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -1822,10 +1822,13 @@ public class DFSOutputStream extends FSOutputSummer
static DFSOutputStream newStreamForAppend(DFSClient dfsClient, String src,
boolean toNewBlock, int bufferSize, Progressable progress,
- LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum)
- throws IOException {
+ LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum,
+ String[] favoredNodes) throws IOException {
final DFSOutputStream out = new DFSOutputStream(dfsClient, src, toNewBlock,
progress, lastBlock, stat, checksum);
+ if (favoredNodes != null && favoredNodes.length != 0) {
+ out.streamer.setFavoredNodes(favoredNodes);
+ }
out.start();
return out;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91a5d929/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 2cecdfb..136ef15 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -317,6 +317,17 @@ public class DistributedFileSystem extends FileSystem {
return append(f, EnumSet.of(CreateFlag.APPEND), bufferSize, progress);
}
+ /**
+ * Append to an existing file (optional operation).
+ *
+ * @param f the existing file to be appended.
+ * @param flag Flags for the Append operation. CreateFlag.APPEND is mandatory
+ * to be present.
+ * @param bufferSize the size of the buffer to be used.
+ * @param progress for reporting progress if it is not null.
+ * @return Returns instance of {@link FSDataOutputStream}
+ * @throws IOException
+ */
public FSDataOutputStream append(Path f, final EnumSet<CreateFlag> flag,
final int bufferSize, final Progressable progress) throws IOException {
statistics.incrementWriteOps(1);
@@ -336,6 +347,38 @@ public class DistributedFileSystem extends FileSystem {
}.resolve(this, absF);
}
+ /**
+ * Append to an existing file (optional operation).
+ *
+ * @param f the existing file to be appended.
+ * @param flag Flags for the Append operation. CreateFlag.APPEND is mandatory
+ * to be present.
+ * @param bufferSize the size of the buffer to be used.
+ * @param progress for reporting progress if it is not null.
+ * @param favoredNodes Favored nodes for new blocks
+ * @return Returns instance of {@link FSDataOutputStream}
+ * @throws IOException
+ */
+ public FSDataOutputStream append(Path f, final EnumSet<CreateFlag> flag,
+ final int bufferSize, final Progressable progress,
+ final InetSocketAddress[] favoredNodes) throws IOException {
+ statistics.incrementWriteOps(1);
+ Path absF = fixRelativePart(f);
+ return new FileSystemLinkResolver<FSDataOutputStream>() {
+ @Override
+ public FSDataOutputStream doCall(final Path p)
+ throws IOException {
+ return dfs.append(getPathName(p), bufferSize, flag, progress,
+ statistics, favoredNodes);
+ }
+ @Override
+ public FSDataOutputStream next(final FileSystem fs, final Path p)
+ throws IOException {
+ return fs.append(p, bufferSize);
+ }
+ }.resolve(this, absF);
+ }
+
@Override
public FSDataOutputStream create(Path f, FsPermission permission,
boolean overwrite, int bufferSize, short replication, long blockSize,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91a5d929/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java
index 4f11037..2d39896 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java
@@ -26,12 +26,14 @@ import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.EnumSet;
import java.util.Random;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -159,6 +161,33 @@ public class TestFavoredNodesEndToEnd {
}
}
+ @Test(timeout = 180000)
+ public void testFavoredNodesEndToEndForAppend() throws Exception {
+ // create 10 files with random preferred nodes
+ for (int i = 0; i < NUM_FILES; i++) {
+ Random rand = new Random(System.currentTimeMillis() + i);
+ // pass a new created rand so as to get a uniform distribution each time
+ // without too much collisions (look at the do-while loop in getDatanodes)
+ InetSocketAddress datanode[] = getDatanodes(rand);
+ Path p = new Path("/filename" + i);
+ // create and close the file.
+ dfs.create(p, FsPermission.getDefault(), true, 4096, (short) 3, 4096L,
+ null, null).close();
+ // re-open for append
+ FSDataOutputStream out = dfs.append(p, EnumSet.of(CreateFlag.APPEND),
+ 4096, null, datanode);
+ out.write(SOME_BYTES);
+ out.close();
+ BlockLocation[] locations = getBlockLocations(p);
+ // verify the files got created in the right nodes
+ for (BlockLocation loc : locations) {
+ String[] hosts = loc.getNames();
+ String[] hosts1 = getStringForInetSocketAddrs(datanode);
+ assertTrue(compareNodes(hosts, hosts1));
+ }
+ }
+ }
+
private BlockLocation[] getBlockLocations(Path p) throws Exception {
DFSTestUtil.waitReplication(dfs, p, (short)3);
BlockLocation[] locations = dfs.getClient().getBlockLocations(
[2/2] hadoop git commit: HADOOP-11000. HAServiceProtocol's health
state is incorrectly transitioned to SERVICE_NOT_RESPONDING (Contributed by
Ming Ma)
Posted by vi...@apache.org.
HADOOP-11000. HAServiceProtocol's health state is incorrectly transitioned to SERVICE_NOT_RESPONDING (Contributed by Ming Ma)
(cherry picked from commit cf4b7f506dd338ecf2ed4c643b6a6a334e070fca)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/005e1df5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/005e1df5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/005e1df5
Branch: refs/heads/branch-2
Commit: 005e1df540aeccbeb371f3bab834b140d85f0ec5
Parents: 91a5d92
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Feb 17 14:55:56 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Feb 17 15:19:44 2015 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/ha/HealthMonitor.java | 35 ++++++----
.../org/apache/hadoop/ha/DummyHAService.java | 73 +++++++++++++++++---
.../org/apache/hadoop/ha/TestHealthMonitor.java | 4 +-
4 files changed, 94 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/005e1df5/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index ff71162..f3ab6f7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -537,6 +537,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11467. KerberosAuthenticator can connect to a non-secure cluster.
(yzhangal via rkanter)
+ HADOOP-11000. HAServiceProtocol's health state is incorrectly transitioned
+ to SERVICE_NOT_RESPONDING (Ming Ma via vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/005e1df5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
index 0d14444..8c87629 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
@@ -30,6 +30,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HealthCheckFailedException;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.util.Daemon;
@@ -201,18 +202,20 @@ public class HealthMonitor {
status = proxy.getServiceStatus();
proxy.monitorHealth();
healthy = true;
- } catch (HealthCheckFailedException e) {
- LOG.warn("Service health check failed for " + targetToMonitor
- + ": " + e.getMessage());
- enterState(State.SERVICE_UNHEALTHY);
} catch (Throwable t) {
- LOG.warn("Transport-level exception trying to monitor health of " +
- targetToMonitor + ": " + t.getLocalizedMessage());
- RPC.stopProxy(proxy);
- proxy = null;
- enterState(State.SERVICE_NOT_RESPONDING);
- Thread.sleep(sleepAfterDisconnectMillis);
- return;
+ if (isHealthCheckFailedException(t)) {
+ LOG.warn("Service health check failed for " + targetToMonitor
+ + ": " + t.getMessage());
+ enterState(State.SERVICE_UNHEALTHY);
+ } else {
+ LOG.warn("Transport-level exception trying to monitor health of " +
+ targetToMonitor + ": " + t.getCause() + " " + t.getLocalizedMessage());
+ RPC.stopProxy(proxy);
+ proxy = null;
+ enterState(State.SERVICE_NOT_RESPONDING);
+ Thread.sleep(sleepAfterDisconnectMillis);
+ return;
+ }
}
if (status != null) {
@@ -225,7 +228,15 @@ public class HealthMonitor {
Thread.sleep(checkIntervalMillis);
}
}
-
+
+ private boolean isHealthCheckFailedException(Throwable t) {
+ return ((t instanceof HealthCheckFailedException) ||
+ (t instanceof RemoteException &&
+ ((RemoteException)t).unwrapRemoteException(
+ HealthCheckFailedException.class) instanceof
+ HealthCheckFailedException));
+ }
+
private synchronized void setLastServiceStatus(HAServiceStatus status) {
this.lastServiceState = status;
for (ServiceStateCallback cb : serviceStateCallbacks) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/005e1df5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java
index e9189e2..aef6c4d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java
@@ -22,15 +22,25 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
+import com.google.protobuf.BlockingService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.ha.protocolPB.HAServiceProtocolPB;
+import org.apache.hadoop.ha.protocolPB.HAServiceProtocolServerSideTranslatorPB;
+import org.apache.hadoop.ha.proto.HAServiceProtocolProtos.HAServiceProtocolService;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
import org.mockito.Mockito;
import com.google.common.collect.Lists;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HA_HM_RPC_TIMEOUT_DEFAULT;
+
/**
* Test-only implementation of {@link HAServiceTarget}, which returns
* a mock implementation.
@@ -50,22 +60,33 @@ class DummyHAService extends HAServiceTarget {
DummySharedResource sharedResource;
public int fenceCount = 0;
public int activeTransitionCount = 0;
+ boolean testWithProtoBufRPC = false;
static ArrayList<DummyHAService> instances = Lists.newArrayList();
int index;
DummyHAService(HAServiceState state, InetSocketAddress address) {
+ this(state, address, false);
+ }
+
+ DummyHAService(HAServiceState state, InetSocketAddress address,
+ boolean testWithProtoBufRPC) {
this.state = state;
- this.proxy = makeMock();
+ this.testWithProtoBufRPC = testWithProtoBufRPC;
+ if (testWithProtoBufRPC) {
+ this.address = startAndGetRPCServerAddress(address);
+ } else {
+ this.address = address;
+ }
+ Configuration conf = new Configuration();
+ this.proxy = makeMock(conf, HA_HM_RPC_TIMEOUT_DEFAULT);
try {
- Configuration conf = new Configuration();
- conf.set(DUMMY_FENCE_KEY, DummyFencer.class.getName());
+ conf.set(DUMMY_FENCE_KEY, DummyFencer.class.getName());
this.fencer = Mockito.spy(
NodeFencer.create(conf, DUMMY_FENCE_KEY));
} catch (BadFencingConfigurationException e) {
throw new RuntimeException(e);
}
- this.address = address;
synchronized (instances) {
instances.add(this);
this.index = instances.size();
@@ -75,9 +96,42 @@ class DummyHAService extends HAServiceTarget {
public void setSharedResource(DummySharedResource rsrc) {
this.sharedResource = rsrc;
}
-
- private HAServiceProtocol makeMock() {
- return Mockito.spy(new MockHAProtocolImpl());
+
+ private InetSocketAddress startAndGetRPCServerAddress(InetSocketAddress serverAddress) {
+ Configuration conf = new Configuration();
+
+ try {
+ RPC.setProtocolEngine(conf,
+ HAServiceProtocolPB.class, ProtobufRpcEngine.class);
+ HAServiceProtocolServerSideTranslatorPB haServiceProtocolXlator =
+ new HAServiceProtocolServerSideTranslatorPB(new MockHAProtocolImpl());
+ BlockingService haPbService = HAServiceProtocolService
+ .newReflectiveBlockingService(haServiceProtocolXlator);
+
+ Server server = new RPC.Builder(conf)
+ .setProtocol(HAServiceProtocolPB.class)
+ .setInstance(haPbService)
+ .setBindAddress(serverAddress.getHostName())
+ .setPort(serverAddress.getPort()).build();
+ server.start();
+ return NetUtils.getConnectAddress(server);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ private HAServiceProtocol makeMock(Configuration conf, int timeoutMs) {
+ HAServiceProtocol service;
+ if (!testWithProtoBufRPC) {
+ service = new MockHAProtocolImpl();
+ } else {
+ try {
+ service = super.getProxy(conf, timeoutMs);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+ return Mockito.spy(service);
}
@Override
@@ -93,6 +147,9 @@ class DummyHAService extends HAServiceTarget {
@Override
public HAServiceProtocol getProxy(Configuration conf, int timeout)
throws IOException {
+ if (testWithProtoBufRPC) {
+ proxy = makeMock(conf, timeout);
+ }
return proxy;
}
@@ -168,7 +225,7 @@ class DummyHAService extends HAServiceTarget {
public HAServiceStatus getServiceStatus() throws IOException {
checkUnreachable();
HAServiceStatus ret = new HAServiceStatus(state);
- if (state == HAServiceState.STANDBY) {
+ if (state == HAServiceState.STANDBY || state == HAServiceState.ACTIVE) {
ret.setReadyToBecomeActive();
}
return ret;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/005e1df5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHealthMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHealthMonitor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHealthMonitor.java
index db534de..b58793f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHealthMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHealthMonitor.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ha;
import static org.junit.Assert.*;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@@ -54,7 +55,8 @@ public class TestHealthMonitor {
conf.setInt(CommonConfigurationKeys.HA_HM_CONNECT_RETRY_INTERVAL_KEY, 50);
conf.setInt(CommonConfigurationKeys.HA_HM_SLEEP_AFTER_DISCONNECT_KEY, 50);
- svc = new DummyHAService(HAServiceState.ACTIVE, null);
+ svc = new DummyHAService(HAServiceState.ACTIVE,
+ new InetSocketAddress("0.0.0.0", 0), true);
hm = new HealthMonitor(conf, svc) {
@Override
protected HAServiceProtocol createProxy() throws IOException {