You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2012/04/19 01:35:38 UTC
svn commit: r1327724 - in
/hadoop/common/branches/HDFS-3042/hadoop-common-project: ./ hadoop-common/
hadoop-common/src/main/bin/ hadoop-common/src/main/docs/
hadoop-common/src/main/java/
hadoop-common/src/main/java/org/apache/hadoop/conf/ hadoop-common...
Author: todd
Date: Wed Apr 18 23:35:30 2012
New Revision: 1327724
URL: http://svn.apache.org/viewvc?rev=1327724&view=rev
Log:
Merge trunk into auto-HA branch
Added:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionUtil.java
- copied unchanged from r1327718, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionUtil.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java
- copied unchanged from r1327718, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/SecurityUtilTestHelper.java
- copied unchanged from r1327718, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/SecurityUtilTestHelper.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
- copied unchanged from r1327718, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestVersionUtil.java
- copied unchanged from r1327718, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestVersionUtil.java
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/ (props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
Propchange: hadoop/common/branches/HDFS-3042/hadoop-common-project/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Wed Apr 18 23:35:30 2012
@@ -1 +1,4 @@
target
+.classpath
+.project
+.settings
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt Wed Apr 18 23:35:30 2012
@@ -58,6 +58,11 @@ Trunk (unreleased changes)
HADOOP-7757. Test file reference count is at least 3x actual value (Jon
Eagles via bobby)
+ HADOOP-8147. test-patch should run tests with -fn to avoid masking test
+ failures (Robert Evans via tgraves)
+
+ HADOOP-8290. Remove remaining references to hadoop.native.lib (harsh)
+
BUG FIXES
HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName.
@@ -205,9 +210,6 @@ Release 2.0.0 - UNRELEASED
HADOOP-7358. Improve log levels when exceptions caught in RPC handler
(Todd Lipcon via shv)
- HADOOP-8108. Move method getHostPortString() from NameNode to NetUtils.
- (Brandon Li via jitendra)
-
HADOOP-7557 Make IPC header be extensible (sanjay radia)
HADOOP-7806. Support binding to sub-interfaces (eli)
@@ -258,6 +260,11 @@ Release 2.0.0 - UNRELEASED
HADOOP-8086. KerberosName silently sets defaultRealm to "" if the
Kerberos config is not found, it should log a WARN (tucu)
+ HADOOP-8280. Move VersionUtil/TestVersionUtil and GenericTestUtils from
+ HDFS into Common. (Ahmed Radwan via atm)
+
+ HADOOP-8117. Upgrade test build to Surefire 2.12 (todd)
+
OPTIMIZATIONS
BUG FIXES
@@ -343,6 +350,16 @@ Release 2.0.0 - UNRELEASED
HADOOP-8264. Remove irritating double double quotes in front of hostname
(Bernd Fondermann via bobby)
+ HADOOP-8270. hadoop-daemon.sh stop action should return 0 for an
+ already stopped service. (Roman Shaposhnik via eli)
+
+ HADOOP-8144. pseudoSortByDistance in NetworkTopology doesn't work
+ properly if no local node and first node is local rack node.
+ (Junping Du)
+
+ HADOOP-8282. start-all.sh refers incorrectly start-dfs.sh
+ existence for starting start-yarn.sh. (Devaraj K via eli)
+
BREAKDOWN OF HADOOP-7454 SUBTASKS
HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh)
@@ -403,6 +420,9 @@ Release 0.23.3 - UNRELEASED
IMPROVEMENTS
+ HADOOP-8108. Move method getHostPortString() from NameNode to NetUtils.
+ (Brandon Li via jitendra)
+
OPTIMIZATIONS
BUG FIXES
@@ -422,6 +442,17 @@ Release 0.23.3 - UNRELEASED
HADOOP-8014. ViewFileSystem does not correctly implement getDefaultBlockSize,
getDefaultReplication, getContentSummary (John George via bobby)
+ HADOOP-7510. Tokens should use original hostname provided instead of ip
+ (Daryn Sharp via bobby)
+
+ HADOOP-8283. Allow tests to control token service value (Daryn Sharp via
+ bobby)
+
+ HADOOP-8286. Simplify getting a socket address from conf (Daryn Sharp via
+ bobby)
+
+ HADOOP-8227. Allow RPC to limit ephemeral port range. (bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1324567-1327718
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh Wed Apr 18 23:35:30 2012
@@ -167,11 +167,9 @@ case $startStop in
kill `cat $pid`
else
echo no $command to stop
- exit 1
fi
else
echo no $command to stop
- exit 1
fi
;;
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh Wed Apr 18 23:35:30 2012
@@ -33,6 +33,6 @@ if [ -f "${HADOOP_HDFS_HOME}"/sbin/start
fi
# start yarn daemons if yarn is present
-if [ -f "${YARN_HOME}"/sbin/start-dfs.sh ]; then
+if [ -f "${YARN_HOME}"/sbin/start-yarn.sh ]; then
"${YARN_HOME}"/sbin/start-yarn.sh --config $HADOOP_CONF_DIR
fi
Propchange: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1324567-1327718
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java Wed Apr 18 23:35:30 2012
@@ -30,6 +30,7 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
+import java.net.InetSocketAddress;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
@@ -68,6 +69,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.codehaus.jackson.JsonFactory;
@@ -962,11 +964,57 @@ public class Configuration implements It
* bound may be omitted meaning all values up to or over. So the string
* above means 2, 3, 5, and 7, 8, 9, ...
*/
- public static class IntegerRanges {
+ public static class IntegerRanges implements Iterable<Integer>{
private static class Range {
int start;
int end;
}
+
+ private static class RangeNumberIterator implements Iterator<Integer> {
+ Iterator<Range> internal;
+ int at;
+ int end;
+
+ public RangeNumberIterator(List<Range> ranges) {
+ if (ranges != null) {
+ internal = ranges.iterator();
+ }
+ at = -1;
+ end = -2;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (at <= end) {
+ return true;
+ } else if (internal != null){
+ return internal.hasNext();
+ }
+ return false;
+ }
+
+ @Override
+ public Integer next() {
+ if (at <= end) {
+ at++;
+ return at - 1;
+ } else if (internal != null){
+ Range found = internal.next();
+ if (found != null) {
+ at = found.start;
+ end = found.end;
+ at++;
+ return at - 1;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
List<Range> ranges = new ArrayList<Range>();
@@ -1025,6 +1073,13 @@ public class Configuration implements It
return false;
}
+ /**
+ * @return true if there are no values in this range, else false.
+ */
+ public boolean isEmpty() {
+ return ranges == null || ranges.isEmpty();
+ }
+
@Override
public String toString() {
StringBuilder result = new StringBuilder();
@@ -1041,6 +1096,12 @@ public class Configuration implements It
}
return result.toString();
}
+
+ @Override
+ public Iterator<Integer> iterator() {
+ return new RangeNumberIterator(ranges);
+ }
+
}
/**
@@ -1163,6 +1224,20 @@ public class Configuration implements It
}
/**
+ * Get the socket address for <code>name</code> property as a
+ * <code>InetSocketAddress</code>.
+ * @param name property name.
+ * @param defaultAddress the default value
+ * @param defaultPort the default port
+ * @return InetSocketAddress
+ */
+ public InetSocketAddress getSocketAddr(
+ String name, String defaultAddress, int defaultPort) {
+ final String address = get(name, defaultAddress);
+ return NetUtils.createSocketAddr(address, defaultPort, name);
+ }
+
+ /**
* Load a class by name.
*
* @param name the class name.
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java Wed Apr 18 23:35:30 2012
@@ -53,7 +53,6 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.ipc.RpcPayloadHeader.*;
import org.apache.hadoop.ipc.protobuf.IpcConnectionContextProtos.IpcConnectionContextProto;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -243,8 +242,8 @@ public class Client {
this.remoteId = remoteId;
this.server = remoteId.getAddress();
if (server.isUnresolved()) {
- throw NetUtils.wrapException(remoteId.getAddress().getHostName(),
- remoteId.getAddress().getPort(),
+ throw NetUtils.wrapException(server.getHostName(),
+ server.getPort(),
null,
0,
new UnknownHostException());
@@ -274,9 +273,8 @@ public class Client {
} catch (IllegalAccessException e) {
throw new IOException(e.toString());
}
- InetSocketAddress addr = remoteId.getAddress();
- token = tokenSelector.selectToken(new Text(addr.getAddress()
- .getHostAddress() + ":" + addr.getPort()),
+ token = tokenSelector.selectToken(
+ SecurityUtil.buildTokenService(server),
ticket.getTokens());
}
KerberosInfo krbInfo = SecurityUtil.getKerberosInfo(protocol, conf);
@@ -305,7 +303,7 @@ public class Client {
+ protocol.getSimpleName());
this.setName("IPC Client (" + socketFactory.hashCode() +") connection to " +
- remoteId.getAddress().toString() +
+ server.toString() +
" from " + ((ticket==null)?"an unknown user":ticket.getUserName()));
this.setDaemon(true);
}
@@ -751,7 +749,6 @@ public class Client {
}
}
- @SuppressWarnings("unused")
public InetSocketAddress getRemoteAddress() {
return server;
}
@@ -1159,7 +1156,7 @@ public class Client {
call.error.fillInStackTrace();
throw call.error;
} else { // local exception
- InetSocketAddress address = remoteId.getAddress();
+ InetSocketAddress address = connection.getRemoteAddress();
throw NetUtils.wrapException(address.getHostName(),
address.getPort(),
NetUtils.getHostname(),
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java Wed Apr 18 23:35:30 2012
@@ -319,10 +319,12 @@ public class ProtobufRpcEngine implement
public RPC.Server getServer(Class<?> protocol, Object protocolImpl,
String bindAddress, int port, int numHandlers, int numReaders,
int queueSizePerHandler, boolean verbose, Configuration conf,
- SecretManager<? extends TokenIdentifier> secretManager)
+ SecretManager<? extends TokenIdentifier> secretManager,
+ String portRangeConfig)
throws IOException {
return new Server(protocol, protocolImpl, conf, bindAddress, port,
- numHandlers, numReaders, queueSizePerHandler, verbose, secretManager);
+ numHandlers, numReaders, queueSizePerHandler, verbose, secretManager,
+ portRangeConfig);
}
public static class Server extends RPC.Server {
@@ -336,15 +338,18 @@ public class ProtobufRpcEngine implement
* @param port the port to listen for connections on
* @param numHandlers the number of method handler threads to run
* @param verbose whether each call should be logged
+ * @param portRangeConfig A config parameter that can be used to restrict
+ * the range of ports used when port is 0 (an ephemeral port)
*/
public Server(Class<?> protocolClass, Object protocolImpl,
Configuration conf, String bindAddress, int port, int numHandlers,
int numReaders, int queueSizePerHandler, boolean verbose,
- SecretManager<? extends TokenIdentifier> secretManager)
+ SecretManager<? extends TokenIdentifier> secretManager,
+ String portRangeConfig)
throws IOException {
super(bindAddress, port, null, numHandlers,
numReaders, queueSizePerHandler, conf, classNameBase(protocolImpl
- .getClass().getName()), secretManager);
+ .getClass().getName()), secretManager, portRangeConfig);
this.verbose = verbose;
registerProtocolAndImpl(RpcKind.RPC_PROTOCOL_BUFFER, protocolClass,
protocolImpl);
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java Wed Apr 18 23:35:30 2012
@@ -654,7 +654,8 @@ public class RPC {
final boolean verbose, Configuration conf)
throws IOException {
return getServer(instance.getClass(), // use impl class for protocol
- instance, bindAddress, port, numHandlers, false, conf, null);
+ instance, bindAddress, port, numHandlers, false, conf, null,
+ null);
}
/** Construct a server for a protocol implementation instance. */
@@ -662,7 +663,8 @@ public class RPC {
Object instance, String bindAddress,
int port, Configuration conf)
throws IOException {
- return getServer(protocol, instance, bindAddress, port, 1, false, conf, null);
+ return getServer(protocol, instance, bindAddress, port, 1, false, conf, null,
+ null);
}
/** Construct a server for a protocol implementation instance.
@@ -676,7 +678,7 @@ public class RPC {
throws IOException {
return getServer(protocol, instance, bindAddress, port, numHandlers, verbose,
- conf, null);
+ conf, null, null);
}
/** Construct a server for a protocol implementation instance. */
@@ -686,10 +688,20 @@ public class RPC {
boolean verbose, Configuration conf,
SecretManager<? extends TokenIdentifier> secretManager)
throws IOException {
-
+ return getServer(protocol, instance, bindAddress, port, numHandlers, verbose,
+ conf, secretManager, null);
+ }
+
+ public static Server getServer(Class<?> protocol,
+ Object instance, String bindAddress, int port,
+ int numHandlers,
+ boolean verbose, Configuration conf,
+ SecretManager<? extends TokenIdentifier> secretManager,
+ String portRangeConfig)
+ throws IOException {
return getProtocolEngine(protocol, conf)
.getServer(protocol, instance, bindAddress, port, numHandlers, -1, -1,
- verbose, conf, secretManager);
+ verbose, conf, secretManager, portRangeConfig);
}
/** Construct a server for a protocol implementation instance. */
@@ -704,7 +716,8 @@ public class RPC {
return getProtocolEngine(protocol, conf)
.getServer(protocol, instance, bindAddress, port, numHandlers,
- numReaders, queueSizePerHandler, verbose, conf, secretManager);
+ numReaders, queueSizePerHandler, verbose, conf, secretManager,
+ null);
}
/** An RPC Server. */
@@ -855,9 +868,10 @@ public class RPC {
Class<? extends Writable> paramClass, int handlerCount,
int numReaders, int queueSizePerHandler,
Configuration conf, String serverName,
- SecretManager<? extends TokenIdentifier> secretManager) throws IOException {
+ SecretManager<? extends TokenIdentifier> secretManager,
+ String portRangeConfig) throws IOException {
super(bindAddress, port, paramClass, handlerCount, numReaders, queueSizePerHandler,
- conf, serverName, secretManager);
+ conf, serverName, secretManager, portRangeConfig);
initProtocolMetaInfo(conf);
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java Wed Apr 18 23:35:30 2012
@@ -47,12 +47,30 @@ public interface RpcEngine {
UserGroupInformation ticket, Configuration conf)
throws IOException, InterruptedException;
- /** Construct a server for a protocol implementation instance. */
+ /**
+ * Construct a server for a protocol implementation instance.
+ *
+ * @param protocol the class of protocol to use
+ * @param instance the instance of protocol whose methods will be called
+ * @param conf the configuration to use
+ * @param bindAddress the address to bind on to listen for connection
+ * @param port the port to listen for connections on
+ * @param numHandlers the number of method handler threads to run
+ * @param numReaders the number of reader threads to run
+ * @param queueSizePerHandler the size of the queue per hander thread
+ * @param verbose whether each call should be logged
+ * @param secretManager The secret manager to use to validate incoming requests.
+ * @param portRangeConfig A config parameter that can be used to restrict
+ * the range of ports used when port is 0 (an ephemeral port)
+ * @return The Server instance
+ * @throws IOException on any error
+ */
RPC.Server getServer(Class<?> protocol, Object instance, String bindAddress,
int port, int numHandlers, int numReaders,
int queueSizePerHandler, boolean verbose,
Configuration conf,
- SecretManager<? extends TokenIdentifier> secretManager
+ SecretManager<? extends TokenIdentifier> secretManager,
+ String portRangeConfig
) throws IOException;
/**
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Wed Apr 18 23:35:30 2012
@@ -63,6 +63,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.BytesWritable;
@@ -291,6 +292,7 @@ public abstract class Server {
protected RpcDetailedMetrics rpcDetailedMetrics;
private Configuration conf;
+ private String portRangeConfig = null;
private SecretManager<TokenIdentifier> secretManager;
private ServiceAuthorizationManager serviceAuthorizationManager = new ServiceAuthorizationManager();
@@ -323,8 +325,33 @@ public abstract class Server {
*/
public static void bind(ServerSocket socket, InetSocketAddress address,
int backlog) throws IOException {
+ bind(socket, address, backlog, null, null);
+ }
+
+ public static void bind(ServerSocket socket, InetSocketAddress address,
+ int backlog, Configuration conf, String rangeConf) throws IOException {
try {
- socket.bind(address, backlog);
+ IntegerRanges range = null;
+ if (rangeConf != null) {
+ range = conf.getRange(rangeConf, "");
+ }
+ if (range == null || range.isEmpty() || (address.getPort() != 0)) {
+ socket.bind(address, backlog);
+ } else {
+ for (Integer port : range) {
+ if (socket.isBound()) break;
+ try {
+ InetSocketAddress temp = new InetSocketAddress(address.getAddress(),
+ port);
+ socket.bind(temp, backlog);
+ } catch(BindException e) {
+ //Ignored
+ }
+ }
+ if (!socket.isBound()) {
+ throw new BindException("Could not find a free port in "+range);
+ }
+ }
} catch (SocketException e) {
throw NetUtils.wrapException(null,
0,
@@ -424,7 +451,7 @@ public abstract class Server {
acceptChannel.configureBlocking(false);
// Bind the server socket to the local host and port
- bind(acceptChannel.socket(), address, backlogLength);
+ bind(acceptChannel.socket(), address, backlogLength, conf, portRangeConfig);
port = acceptChannel.socket().getLocalPort(); //Could be an ephemeral port
// create a selector;
selector= Selector.open();
@@ -1725,7 +1752,16 @@ public abstract class Server {
throws IOException
{
this(bindAddress, port, paramClass, handlerCount, -1, -1, conf, Integer
- .toString(port), null);
+ .toString(port), null, null);
+ }
+
+ protected Server(String bindAddress, int port,
+ Class<? extends Writable> rpcRequestClass, int handlerCount,
+ int numReaders, int queueSizePerHandler, Configuration conf,
+ String serverName, SecretManager<? extends TokenIdentifier> secretManager)
+ throws IOException {
+ this(bindAddress, port, rpcRequestClass, handlerCount, numReaders,
+ queueSizePerHandler, conf, serverName, secretManager, null);
}
/**
@@ -1745,10 +1781,12 @@ public abstract class Server {
protected Server(String bindAddress, int port,
Class<? extends Writable> rpcRequestClass, int handlerCount,
int numReaders, int queueSizePerHandler, Configuration conf,
- String serverName, SecretManager<? extends TokenIdentifier> secretManager)
+ String serverName, SecretManager<? extends TokenIdentifier> secretManager,
+ String portRangeConfig)
throws IOException {
this.bindAddress = bindAddress;
this.conf = conf;
+ this.portRangeConfig = portRangeConfig;
this.port = port;
this.rpcRequestClass = rpcRequestClass;
this.handlerCount = handlerCount;
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java Wed Apr 18 23:35:30 2012
@@ -299,16 +299,19 @@ public class WritableRpcEngine implement
}
}
- /** Construct a server for a protocol implementation instance listening on a
+ /* Construct a server for a protocol implementation instance listening on a
* port and address. */
+ @Override
public RPC.Server getServer(Class<?> protocolClass,
Object protocolImpl, String bindAddress, int port,
int numHandlers, int numReaders, int queueSizePerHandler,
boolean verbose, Configuration conf,
- SecretManager<? extends TokenIdentifier> secretManager)
+ SecretManager<? extends TokenIdentifier> secretManager,
+ String portRangeConfig)
throws IOException {
return new Server(protocolClass, protocolImpl, conf, bindAddress, port,
- numHandlers, numReaders, queueSizePerHandler, verbose, secretManager);
+ numHandlers, numReaders, queueSizePerHandler, verbose, secretManager,
+ portRangeConfig);
}
@@ -341,7 +344,7 @@ public class WritableRpcEngine implement
Configuration conf, String bindAddress, int port)
throws IOException {
this(protocolClass, protocolImpl, conf, bindAddress, port, 1, -1, -1,
- false, null);
+ false, null, null);
}
/**
@@ -363,7 +366,7 @@ public class WritableRpcEngine implement
throws IOException {
this(null, protocolImpl, conf, bindAddress, port,
numHandlers, numReaders, queueSizePerHandler, verbose,
- secretManager);
+ secretManager, null);
}
@@ -381,11 +384,13 @@ public class WritableRpcEngine implement
public Server(Class<?> protocolClass, Object protocolImpl,
Configuration conf, String bindAddress, int port,
int numHandlers, int numReaders, int queueSizePerHandler,
- boolean verbose, SecretManager<? extends TokenIdentifier> secretManager)
+ boolean verbose, SecretManager<? extends TokenIdentifier> secretManager,
+ String portRangeConfig)
throws IOException {
super(bindAddress, port, null, numHandlers, numReaders,
queueSizePerHandler, conf,
- classNameBase(protocolImpl.getClass().getName()), secretManager);
+ classNameBase(protocolImpl.getClass().getName()), secretManager,
+ portRangeConfig);
this.verbose = verbose;
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java Wed Apr 18 23:35:30 2012
@@ -344,8 +344,8 @@ public class NetUtils {
/**
* Returns InetSocketAddress that a client can use to
* connect to the server. Server.getListenerAddress() is not correct when
- * the server binds to "0.0.0.0". This returns "127.0.0.1:port" when
- * the getListenerAddress() returns "0.0.0.0:port".
+ * the server binds to "0.0.0.0". This returns "hostname:port" of the server,
+ * or "127.0.0.1:port" when the getListenerAddress() returns "0.0.0.0:port".
*
* @param server
* @return socket address that a client can use to connect to the server.
@@ -353,7 +353,12 @@ public class NetUtils {
public static InetSocketAddress getConnectAddress(Server server) {
InetSocketAddress addr = server.getListenerAddress();
if (addr.getAddress().isAnyLocalAddress()) {
- addr = createSocketAddrForHost("127.0.0.1", addr.getPort());
+ try {
+ addr = new InetSocketAddress(InetAddress.getLocalHost(), addr.getPort());
+ } catch (UnknownHostException uhe) {
+ // shouldn't get here unless the host doesn't have a loopback iface
+ addr = createSocketAddrForHost("127.0.0.1", addr.getPort());
+ }
}
return addr;
}
@@ -655,7 +660,7 @@ public class NetUtils {
}
InetAddress addr = null;
try {
- addr = InetAddress.getByName(host);
+ addr = SecurityUtil.getByName(host);
if (NetworkInterface.getByInetAddress(addr) == null) {
addr = null; // Not a local address
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java Wed Apr 18 23:35:30 2012
@@ -662,8 +662,8 @@ public class NetworkTopology {
*/
public void pseudoSortByDistance( Node reader, Node[] nodes ) {
int tempIndex = 0;
+ int localRackNode = -1;
if (reader != null ) {
- int localRackNode = -1;
//scan the array to find the local node & local rack node
for(int i=0; i<nodes.length; i++) {
if(tempIndex == 0 && reader == nodes[i]) { //local node
@@ -693,7 +693,7 @@ public class NetworkTopology {
}
// put a random node at position 0 if it is not a local/local-rack node
- if(tempIndex == 0 && nodes.length != 0) {
+ if(tempIndex == 0 && localRackNode == -1 && nodes.length != 0) {
swap(nodes, 0, r.nextInt(nodes.length));
}
}
Propchange: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1324567-1327718
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java Wed Apr 18 23:35:30 2012
@@ -23,18 +23,24 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringWriter;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import java.util.regex.Pattern;
import junit.framework.TestCase;
import static org.junit.Assert.assertArrayEquals;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.net.NetUtils;
import org.codehaus.jackson.map.ObjectMapper;
public class TestConfiguration extends TestCase {
@@ -360,6 +366,35 @@ public class TestConfiguration extends T
assertEquals(true, range.isIncluded(34));
assertEquals(true, range.isIncluded(100000000));
}
+
+ public void testGetRangeIterator() throws Exception {
+ Configuration config = new Configuration(false);
+ IntegerRanges ranges = config.getRange("Test", "");
+ assertFalse("Empty range has values", ranges.iterator().hasNext());
+ ranges = config.getRange("Test", "5");
+ Set<Integer> expected = new HashSet<Integer>(Arrays.asList(5));
+ Set<Integer> found = new HashSet<Integer>();
+ for(Integer i: ranges) {
+ found.add(i);
+ }
+ assertEquals(expected, found);
+
+ ranges = config.getRange("Test", "5-10,13-14");
+ expected = new HashSet<Integer>(Arrays.asList(5,6,7,8,9,10,13,14));
+ found = new HashSet<Integer>();
+ for(Integer i: ranges) {
+ found.add(i);
+ }
+ assertEquals(expected, found);
+
+ ranges = config.getRange("Test", "8-12, 5- 7");
+ expected = new HashSet<Integer>(Arrays.asList(5,6,7,8,9,10,11,12));
+ found = new HashSet<Integer>();
+ for(Integer i: ranges) {
+ found.add(i);
+ }
+ assertEquals(expected, found);
+ }
public void testHexValues() throws IOException{
out=new BufferedWriter(new FileWriter(CONFIG));
@@ -604,6 +639,38 @@ public class TestConfiguration extends T
conf.getPattern("test.pattern3", defaultPattern).pattern());
}
+ public void testSocketAddress() throws IOException {
+ Configuration conf = new Configuration();
+ final String defaultAddr = "host:1";
+ final int defaultPort = 2;
+ InetSocketAddress addr = null;
+
+ addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
+ assertEquals(defaultAddr, NetUtils.getHostPortString(addr));
+
+ conf.set("myAddress", "host2");
+ addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
+ assertEquals("host2:"+defaultPort, NetUtils.getHostPortString(addr));
+
+ conf.set("myAddress", "host2:3");
+ addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
+ assertEquals("host2:3", NetUtils.getHostPortString(addr));
+
+ boolean threwException = false;
+ conf.set("myAddress", "bad:-port");
+ try {
+ addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
+ } catch (IllegalArgumentException iae) {
+ threwException = true;
+ assertEquals("Does not contain a valid host:port authority: " +
+ "bad:-port (configuration property 'myAddress')",
+ iae.getMessage());
+
+ } finally {
+ assertTrue(threwException);
+ }
+ }
+
public void testReload() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java Wed Apr 18 23:35:30 2012
@@ -756,7 +756,8 @@ public class TestCodec {
// Don't use native libs for this test.
Configuration conf = new Configuration();
- conf.setBoolean("hadoop.native.lib", false);
+ conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
+ false);
assertFalse("ZlibFactory is using native libs against request",
ZlibFactory.isNativeZlibLoaded(conf));
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java Wed Apr 18 23:35:30 2012
@@ -33,6 +33,7 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -237,7 +238,8 @@ public class TestTFileSeqFileComparison
public SeqFileAppendable(FileSystem fs, Path path, int osBufferSize,
String compress, int minBlkSize) throws IOException {
Configuration conf = new Configuration();
- conf.setBoolean("hadoop.native.lib", true);
+ conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
+ true);
CompressionCodec codec = null;
if ("lzo".equals(compress)) {
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java Wed Apr 18 23:35:30 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.ipc;
import java.io.Closeable;
import java.io.IOException;
import java.net.ConnectException;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
@@ -41,6 +42,8 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryProxy;
import org.apache.hadoop.ipc.Client.ConnectionId;
+import org.apache.hadoop.ipc.TestSaslRPC.TestSaslImpl;
+import org.apache.hadoop.ipc.TestSaslRPC.TestSaslProtocol;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authorize.AuthorizationException;
@@ -257,7 +260,8 @@ public class TestRPC {
public org.apache.hadoop.ipc.RPC.Server getServer(Class<?> protocol,
Object instance, String bindAddress, int port, int numHandlers,
int numReaders, int queueSizePerHandler, boolean verbose, Configuration conf,
- SecretManager<? extends TokenIdentifier> secretManager) throws IOException {
+ SecretManager<? extends TokenIdentifier> secretManager,
+ String portRangeConfig) throws IOException {
return null;
}
@@ -543,6 +547,19 @@ public class TestRPC {
}
@Test
+ public void testServerAddress() throws IOException {
+ Server server = RPC.getServer(TestProtocol.class,
+ new TestImpl(), ADDRESS, 0, 5, true, conf, null);
+ InetSocketAddress bindAddr = null;
+ try {
+ bindAddr = NetUtils.getConnectAddress(server);
+ } finally {
+ server.stop();
+ }
+ assertEquals(bindAddr.getAddress(), InetAddress.getLocalHost());
+ }
+
+ @Test
public void testAuthorization() throws Exception {
Configuration conf = new Configuration();
conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,