You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2012/05/03 04:14:23 UTC
svn commit: r1333291 [1/2] - in
/hadoop/common/branches/HDFS-3042/hadoop-common-project: dev-support/
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/
hadoop-common/ hadoop-common/dev-support/ hadoop-common/src/main/bin/ hado...
Author: todd
Date: Thu May 3 02:14:01 2012
New Revision: 1333291
URL: http://svn.apache.org/viewvc?rev=1333291&view=rev
Log:
Merge trunk into auto-HA branch
Added:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
- copied unchanged from r1333288, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/proto/RpcPayloadHeader.proto
- copied unchanged from r1333288, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/proto/RpcPayloadHeader.proto
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
- copied unchanged from r1333288, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec
- copied unchanged from r1333288, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
- copied unchanged from r1333288, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
- copied unchanged from r1333288, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
Removed:
hadoop/common/branches/HDFS-3042/hadoop-common-project/dev-support/test-patch.properties
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcPayloadHeader.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/resources/test-patch.properties
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/commands_manual.xml
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlStreamHandlerFactory.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FenceMethod.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInfoServerSideTranslatorPB.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationKey.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestDeprecatedKeys.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextDeleteOnExit.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableUtils.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPCServerResponder.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java Thu May 3 02:14:01 2012
@@ -288,7 +288,7 @@ public class KerberosAuthenticationHandl
String clientPrincipal = gssContext.getSrcName().toString();
KerberosName kerberosName = new KerberosName(clientPrincipal);
String userName = kerberosName.getShortName();
- token = new AuthenticationToken(userName, clientPrincipal, TYPE);
+ token = new AuthenticationToken(userName, clientPrincipal, getType());
response.setStatus(HttpServletResponse.SC_OK);
LOG.trace("SPNEGO completed for principal [{}]", clientPrincipal);
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java Thu May 3 02:14:01 2012
@@ -126,7 +126,7 @@ public class PseudoAuthenticationHandler
throw new AuthenticationException("Anonymous requests are disallowed");
}
} else {
- token = new AuthenticationToken(userName, userName, TYPE);
+ token = new AuthenticationToken(userName, userName, getType());
}
return token;
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt Thu May 3 02:14:01 2012
@@ -63,6 +63,10 @@ Trunk (unreleased changes)
HADOOP-8290. Remove remaining references to hadoop.native.lib (harsh)
+ HADOOP-8285 Use ProtoBuf for RpcPayLoadHeader (sanjay radia)
+
+ HADOOP-8308. Support cross-project Jenkins builds. (tomwhite)
+
BUG FIXES
HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName.
@@ -119,6 +123,12 @@ Trunk (unreleased changes)
HADOOP-7788. Add simple HealthMonitor class to watch an HAService (todd)
+ HADOOP-8312. testpatch.sh should provide a simpler way to see which
+ warnings changed (bobby)
+
+ HADOOP-8339. jenkins complaining about 16 javadoc warnings
+ (Tom White and Robert Evans via tgraves)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -265,6 +275,15 @@ Release 2.0.0 - UNRELEASED
HADOOP-8117. Upgrade test build to Surefire 2.12 (todd)
+ HADOOP-8152. Expand public APIs for security library classes. (atm via eli)
+
+ HADOOP-7549. Use JDK ServiceLoader mechanism to find FileSystem implementations. (tucu)
+
+ HADOOP-8185. Update namenode -format documentation and add -nonInteractive
+ and -force. (Arpit Gupta via atm)
+
+ HADOOP-8214. make hadoop script recognize a full set of deprecated commands (rvs via tucu)
+
OPTIMIZATIONS
BUG FIXES
@@ -360,6 +379,33 @@ Release 2.0.0 - UNRELEASED
HADOOP-8282. start-all.sh refers incorrectly start-dfs.sh
existence for starting start-yarn.sh. (Devaraj K via eli)
+ HADOOP-7350. Use ServiceLoader to discover compression codec classes.
+ (tomwhite)
+
+ HADOOP-8284. clover integration broken, also mapreduce poms are pulling
+ in clover as a dependency. (phunt via tucu)
+
+ HADOOP-8309. Pseudo & Kerberos AuthenticationHandler should use
+ getType() to create token (tucu)
+
+ HADOOP-8314. HttpServer#hasAdminAccess should return false if
+ authorization is enabled but user is not authenticated. (tucu)
+
+ HADOOP-8296. hadoop/yarn daemonlog usage wrong (Devaraj K via tgraves)
+
+ HADOOP-8310. FileContext#checkPath should handle URIs with no port. (atm)
+
+ HADOOP-8321. TestUrlStreamHandler fails. (tucu)
+
+ HADOOP-8325. Add a ShutdownHookManager to be used by different
+ components instead of the JVM shutdownhook (tucu)
+
+ HADOOP-8275. Range check DelegationKey length.
+ (Colin Patrick McCabe via eli)
+
+ HADOOP-8342. HDFS command fails with exception following merge of
+ HADOOP-8325 (tucu)
+
BREAKDOWN OF HADOOP-7454 SUBTASKS
HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh)
@@ -412,6 +458,12 @@ Release 2.0.0 - UNRELEASED
HADOOP-8116. RetriableCommand is using RetryPolicy incorrectly after
HADOOP-7896. (atm)
+ HADOOP-8317. Update maven-assembly-plugin to 2.3 - fix build on FreeBSD
+ (Radim Kolar via bobby)
+
+ HADOOP-8172. Configuration no longer sets all keys in a deprecated key
+ list. (Anupam Seth via bobby)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -423,6 +475,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8108. Move method getHostPortString() from NameNode to NetUtils.
(Brandon Li via jitendra)
+ HADOOP-8288. Remove references of mapred.child.ulimit etc. since they are
+ not being used any more (Ravi Prakash via bobby)
+
OPTIMIZATIONS
BUG FIXES
@@ -453,6 +508,17 @@ Release 0.23.3 - UNRELEASED
HADOOP-8227. Allow RPC to limit ephemeral port range. (bobby)
+ HADOOP-8305. distcp over viewfs is broken (John George via bobby)
+
+ HADOOP-8334. HttpServer sometimes returns incorrect port (Daryn Sharp via
+ bobby)
+
+ HADOOP-8330. Update TestSequenceFile.testCreateUsesFsArg() for HADOOP-8305.
+ (John George via szetszwo)
+
+ HADOOP-8335. Improve Configuration's address handling (Daryn Sharp via
+ bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1327719-1333290
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml Thu May 3 02:14:01 2012
@@ -282,8 +282,13 @@
<!-- protobuf generated code -->
<Class name="~org\.apache\.hadoop\.ipc\.protobuf\.IpcConnectionContextProtos.*"/>
</Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.ipc\.protobuf\.RpcPayloadHeaderProtos.*"/>
+ </Match>
<Match>
<!-- protobuf generated code -->
<Class name="~org\.apache\.hadoop\.ha\.proto\.HAServiceProtocolProtos.*"/>
</Match>
+
</FindBugsFilter>
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop Thu May 3 02:14:01 2012
@@ -50,15 +50,16 @@ fi
COMMAND=$1
case $COMMAND in
#hdfs commands
- namenode|secondarynamenode|datanode|dfs|dfsadmin|fsck|balancer|fetchdt)
+ namenode|secondarynamenode|datanode|dfs|dfsadmin|fsck|balancer|fetchdt|oiv|dfsgroups)
echo "DEPRECATED: Use of this script to execute hdfs command is deprecated." 1>&2
echo "Instead use the hdfs command for it." 1>&2
echo "" 1>&2
#try to locate hdfs and if present, delegate to it.
+ shift
if [ -f "${HADOOP_HDFS_HOME}"/bin/hdfs ]; then
- exec "${HADOOP_HDFS_HOME}"/bin/hdfs $*
+ exec "${HADOOP_HDFS_HOME}"/bin/hdfs ${COMMAND/dfsgroups/groups} $*
elif [ -f "${HADOOP_PREFIX}"/bin/hdfs ]; then
- exec "${HADOOP_PREFIX}"/bin/hdfs $*
+ exec "${HADOOP_PREFIX}"/bin/hdfs ${COMMAND/dfsgroups/groups} $*
else
echo "HADOOP_HDFS_HOME not found!"
exit 1
@@ -66,15 +67,16 @@ case $COMMAND in
;;
#mapred commands for backwards compatibility
- pipes|job|queue)
+ pipes|job|queue|mrgroups|mradmin|jobtracker|tasktracker)
echo "DEPRECATED: Use of this script to execute mapred command is deprecated." 1>&2
echo "Instead use the mapred command for it." 1>&2
echo "" 1>&2
#try to locate mapred and if present, delegate to it.
+ shift
if [ -f "${HADOOP_MAPRED_HOME}"/bin/mapred ]; then
- exec "${HADOOP_MAPRED_HOME}"/bin/mapred $*
+ exec "${HADOOP_MAPRED_HOME}"/bin/mapred ${COMMAND/mrgroups/groups} $*
elif [ -f "${HADOOP_PREFIX}"/bin/mapred ]; then
- exec "${HADOOP_PREFIX}"/bin/mapred $*
+ exec "${HADOOP_PREFIX}"/bin/mapred ${COMMAND/mrgroups/groups} $*
else
echo "HADOOP_MAPRED_HOME not found!"
exit 1
Propchange: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1327719-1333290
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/commands_manual.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/commands_manual.xml?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/commands_manual.xml (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/commands_manual.xml Thu May 3 02:14:01 2012
@@ -696,7 +696,7 @@
<a href="http://hadoop.apache.org/hdfs/docs/current/hdfs_user_guide.html#Upgrade+and+Rollback">Upgrade and Rollback</a>.
</p>
<p>
- <code>Usage: hadoop namenode [-format] | [-upgrade] | [-rollback] | [-finalize] | [-importCheckpoint] | [-checkpoint] | [-backup]</code>
+ <code>Usage: hadoop namenode [-format [-force] [-nonInteractive] [-clusterid someid]] | [-upgrade] | [-rollback] | [-finalize] | [-importCheckpoint] | [-checkpoint] | [-backup]</code>
</p>
<table>
<tr><th> COMMAND_OPTION </th><th> Description </th></tr>
@@ -714,8 +714,11 @@
<td>Start namenode in backup role, maintaining an up-to-date in-memory copy of the namespace and creating periodic checkpoints.</td>
</tr>
<tr>
- <td><code>-format</code></td>
- <td>Formats the namenode. It starts the namenode, formats it and then shut it down.</td>
+ <td><code>-format [-force] [-nonInteractive] [-clusterid someid]</code></td>
+ <td>Formats the namenode. It starts the namenode, formats it and then shuts it down. User will be prompted before formatting any non empty name directories in the local filesystem.<br/>
+ -nonInteractive: User will not be prompted for input if non empty name directories exist in the local filesystem and the format will fail.<br/>
+ -force: Formats the namenode and the user will NOT be prompted to confirm formatting of the name directories in the local filesystem. If -nonInteractive option is specified it will be ignored.<br/>
+ -clusterid: Associates the namenode with the id specified. When formatting federated namenodes use this option to make sure all namenodes are associated with the same id.</td>
</tr>
<tr>
<td><code>-upgrade</code></td>
Propchange: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1327719-1333290
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java Thu May 3 02:14:01 2012
@@ -33,6 +33,7 @@ import java.io.Writer;
import java.net.InetSocketAddress;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
@@ -269,10 +270,18 @@ public class Configuration implements It
* This is to be used only by the developers in order to add deprecation of
* keys, and attempts to call this method after loading resources once,
* would lead to <tt>UnsupportedOperationException</tt>
+ *
+ * If a key is deprecated in favor of multiple keys, they are all treated as
+ * aliases of each other, and setting any one of them resets all the others
+ * to the new value.
+ *
* @param key
* @param newKeys
* @param customMessage
+ * @deprecated use {@link addDeprecation(String key, String newKey,
+ String customMessage)} instead
*/
+ @Deprecated
public synchronized static void addDeprecation(String key, String[] newKeys,
String customMessage) {
if (key == null || key.length() == 0 ||
@@ -288,6 +297,22 @@ public class Configuration implements It
}
}
}
+
+ /**
+ * Adds the deprecated key to the deprecation map.
+ * It does not override any existing entries in the deprecation map.
+ * This is to be used only by the developers in order to add deprecation of
+ * keys, and attempts to call this method after loading resources once,
+ * would lead to <tt>UnsupportedOperationException</tt>
+ *
+ * @param key
+ * @param newKey
+ * @param customMessage
+ */
+ public synchronized static void addDeprecation(String key, String newKey,
+ String customMessage) {
+ addDeprecation(key, new String[] {newKey}, customMessage);
+ }
/**
* Adds the deprecated key to the deprecation map when no custom message
@@ -297,14 +322,35 @@ public class Configuration implements It
* keys, and attempts to call this method after loading resources once,
* would lead to <tt>UnsupportedOperationException</tt>
*
+ * If a key is deprecated in favor of multiple keys, they are all treated as
+ * aliases of each other, and setting any one of them resets all the others
+ * to the new value.
+ *
* @param key Key that is to be deprecated
* @param newKeys list of keys that take up the values of deprecated key
+ * @deprecated use {@link addDeprecation(String key, String newKey)} instead
*/
+ @Deprecated
public synchronized static void addDeprecation(String key, String[] newKeys) {
addDeprecation(key, newKeys, null);
}
/**
+ * Adds the deprecated key to the deprecation map when no custom message
+ * is provided.
+ * It does not override any existing entries in the deprecation map.
+ * This is to be used only by the developers in order to add deprecation of
+ * keys, and attempts to call this method after loading resources once,
+ * would lead to <tt>UnsupportedOperationException</tt>
+ *
+ * @param key Key that is to be deprecated
+ * @param newKey key that takes up the value of deprecated key
+ */
+ public synchronized static void addDeprecation(String key, String newKey) {
+ addDeprecation(key, new String[] {newKey}, null);
+ }
+
+ /**
* checks whether the given <code>key</code> is deprecated.
*
* @param key the parameter which is to be checked for deprecation
@@ -322,16 +368,26 @@ public class Configuration implements It
* @param name property name.
* @return alternate name.
*/
- private String getAlternateName(String name) {
- String altName;
+ private String[] getAlternateNames(String name) {
+ String oldName, altNames[] = null;
DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
- if (keyInfo != null) {
- altName = (keyInfo.newKeys.length > 0) ? keyInfo.newKeys[0] : null;
- }
- else {
- altName = reverseDeprecatedKeyMap.get(name);
+ if (keyInfo == null) {
+ altNames = (reverseDeprecatedKeyMap.get(name) != null ) ?
+ new String [] {reverseDeprecatedKeyMap.get(name)} : null;
+ if(altNames != null && altNames.length > 0) {
+ //To help look for other new configs for this deprecated config
+ keyInfo = deprecatedKeyMap.get(altNames[0]);
+ }
+ }
+ if(keyInfo != null && keyInfo.newKeys.length > 0) {
+ List<String> list = new ArrayList<String>();
+ if(altNames != null) {
+ list.addAll(Arrays.asList(altNames));
+ }
+ list.addAll(Arrays.asList(keyInfo.newKeys));
+ altNames = list.toArray(new String[list.size()]);
}
- return altName;
+ return altNames;
}
/**
@@ -346,24 +402,29 @@ public class Configuration implements It
* @return the first property in the list of properties mapping
* the <code>name</code> or the <code>name</code> itself.
*/
- private String handleDeprecation(String name) {
- if (isDeprecated(name)) {
+ private String[] handleDeprecation(String name) {
+ ArrayList<String > names = new ArrayList<String>();
+ if (isDeprecated(name)) {
DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
warnOnceIfDeprecated(name);
for (String newKey : keyInfo.newKeys) {
if(newKey != null) {
- name = newKey;
- break;
+ names.add(newKey);
}
}
}
- String deprecatedKey = reverseDeprecatedKeyMap.get(name);
- if (deprecatedKey != null && !getOverlay().containsKey(name) &&
- getOverlay().containsKey(deprecatedKey)) {
- getProps().setProperty(name, getOverlay().getProperty(deprecatedKey));
- getOverlay().setProperty(name, getOverlay().getProperty(deprecatedKey));
+ if(names.size() == 0) {
+ names.add(name);
+ }
+ for(String n : names) {
+ String deprecatedKey = reverseDeprecatedKeyMap.get(n);
+ if (deprecatedKey != null && !getOverlay().containsKey(n) &&
+ getOverlay().containsKey(deprecatedKey)) {
+ getProps().setProperty(n, getOverlay().getProperty(deprecatedKey));
+ getOverlay().setProperty(n, getOverlay().getProperty(deprecatedKey));
+ }
}
- return name;
+ return names.toArray(new String[names.size()]);
}
private void handleDeprecation() {
@@ -595,8 +656,12 @@ public class Configuration implements It
* or null if no such property exists.
*/
public String get(String name) {
- name = handleDeprecation(name);
- return substituteVars(getProps().getProperty(name));
+ String[] names = handleDeprecation(name);
+ String result = null;
+ for(String n : names) {
+ result = substituteVars(getProps().getProperty(n));
+ }
+ return result;
}
/**
@@ -633,8 +698,12 @@ public class Configuration implements It
* its replacing property and null if no such property exists.
*/
public String getRaw(String name) {
- name = handleDeprecation(name);
- return getProps().getProperty(name);
+ String[] names = handleDeprecation(name);
+ String result = null;
+ for(String n : names) {
+ result = getProps().getProperty(n);
+ }
+ return result;
}
/**
@@ -652,10 +721,12 @@ public class Configuration implements It
getOverlay().setProperty(name, value);
getProps().setProperty(name, value);
updatingResource.put(name, UNKNOWN_RESOURCE);
- String altName = getAlternateName(name);
- if (altName != null) {
- getOverlay().setProperty(altName, value);
- getProps().setProperty(altName, value);
+ String[] altNames = getAlternateNames(name);
+ if (altNames != null && altNames.length > 0) {
+ for(String altName : altNames) {
+ getOverlay().setProperty(altName, value);
+ getProps().setProperty(altName, value);
+ }
}
warnOnceIfDeprecated(name);
}
@@ -671,12 +742,14 @@ public class Configuration implements It
* Unset a previously set property.
*/
public synchronized void unset(String name) {
- String altName = getAlternateName(name);
+ String[] altNames = getAlternateNames(name);
getOverlay().remove(name);
getProps().remove(name);
- if (altName !=null) {
- getOverlay().remove(altName);
- getProps().remove(altName);
+ if (altNames !=null && altNames.length > 0) {
+ for(String altName : altNames) {
+ getOverlay().remove(altName);
+ getProps().remove(altName);
+ }
}
}
@@ -711,8 +784,12 @@ public class Configuration implements It
* doesn't exist.
*/
public String get(String name, String defaultValue) {
- name = handleDeprecation(name);
- return substituteVars(getProps().getProperty(name, defaultValue));
+ String[] names = handleDeprecation(name);
+ String result = null;
+ for(String n : names) {
+ result = substituteVars(getProps().getProperty(n, defaultValue));
+ }
+ return result;
}
/**
@@ -1236,6 +1313,29 @@ public class Configuration implements It
final String address = get(name, defaultAddress);
return NetUtils.createSocketAddr(address, defaultPort, name);
}
+
+ /**
+ * Set the socket address for the <code>name</code> property as
+ * a <code>host:port</code>.
+ */
+ public void setSocketAddr(String name, InetSocketAddress addr) {
+ set(name, NetUtils.getHostPortString(addr));
+ }
+
+ /**
+ * Set the socket address a client can use to connect for the
+ * <code>name</code> property as a <code>host:port</code>. The wildcard
+ * address is replaced with the local host's address.
+ * @param name property name.
+ * @param addr InetSocketAddress of a listener to store in the given property
+ * @return InetSocketAddress for clients to connect
+ */
+ public InetSocketAddress updateConnectAddr(String name,
+ InetSocketAddress addr) {
+ final InetSocketAddress connectAddr = NetUtils.getConnectAddress(addr);
+ setSocketAddr(name, connectAddr);
+ return connectAddr;
+ }
/**
* Load a class by name.
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java Thu May 3 02:14:01 2012
@@ -350,20 +350,23 @@ public abstract class AbstractFileSystem
}
}
String thisScheme = this.getUri().getScheme();
- String thisAuthority = this.getUri().getAuthority();
+ String thisHost = this.getUri().getHost();
+ String thatHost = uri.getHost();
- // Schemes and authorities must match.
+ // Schemes and hosts must match.
// Allow for null Authority for file:///
if (!thisScheme.equalsIgnoreCase(thatScheme) ||
- (thisAuthority != null &&
- !thisAuthority.equalsIgnoreCase(thatAuthority)) ||
- (thisAuthority == null && thatAuthority != null)) {
+ (thisHost != null &&
+ !thisHost.equalsIgnoreCase(thatHost)) ||
+ (thisHost == null && thatHost != null)) {
throw new InvalidPathException("Wrong FS: " + path + ", expected: "
+ this.getUri());
}
+ // Ports must match, unless this FS instance is using the default port, in
+ // which case the port may be omitted from the given URI
int thisPort = this.getUri().getPort();
- int thatPort = path.toUri().getPort();
+ int thatPort = uri.getPort();
if (thatPort == -1) { // -1 => defaultPort of Uri scheme
thatPort = this.getUriDefaultPort();
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java Thu May 3 02:14:01 2012
@@ -54,6 +54,7 @@ import org.apache.hadoop.fs.InvalidPathE
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ShutdownHookManager;
/**
* The FileContext class provides an interface to the application writer for
@@ -171,7 +172,12 @@ public final class FileContext {
public static final Log LOG = LogFactory.getLog(FileContext.class);
public static final FsPermission DEFAULT_PERM = FsPermission.getDefault();
-
+
+ /**
+ * Priority of the FileContext shutdown hook.
+ */
+ public static final int SHUTDOWN_HOOK_PRIORITY = 20;
+
/**
* List of files that should be deleted on JVM shutdown.
*/
@@ -1456,8 +1462,8 @@ public final class FileContext {
return false;
}
synchronized (DELETE_ON_EXIT) {
- if (DELETE_ON_EXIT.isEmpty() && !FINALIZER.isAlive()) {
- Runtime.getRuntime().addShutdownHook(FINALIZER);
+ if (DELETE_ON_EXIT.isEmpty()) {
+ ShutdownHookManager.get().addShutdownHook(FINALIZER, SHUTDOWN_HOOK_PRIORITY);
}
Set<Path> set = DELETE_ON_EXIT.get(this);
@@ -2215,7 +2221,7 @@ public final class FileContext {
/**
* Deletes all the paths in deleteOnExit on JVM shutdown.
*/
- static class FileContextFinalizer extends Thread {
+ static class FileContextFinalizer implements Runnable {
public synchronized void run() {
processDeleteOnExit();
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Thu May 3 02:14:01 2012
@@ -32,6 +32,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.ServiceLoader;
import java.util.Set;
import java.util.Stack;
import java.util.TreeSet;
@@ -54,6 +55,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.ShutdownHookManager;
/****************************************************************
* An abstract base class for a fairly generic filesystem. It
@@ -83,6 +85,11 @@ public abstract class FileSystem extends
public static final Log LOG = LogFactory.getLog(FileSystem.class);
+ /**
+ * Priority of the FileSystem shutdown hook.
+ */
+ public static final int SHUTDOWN_HOOK_PRIORITY = 10;
+
/** FileSystem cache */
static final Cache CACHE = new Cache();
@@ -184,6 +191,17 @@ public abstract class FileSystem extends
statistics = getStatistics(name.getScheme(), getClass());
}
+ /**
+ * Return the protocol scheme for the FileSystem.
+ * <p/>
+ * This implementation throws an <code>UnsupportedOperationException</code>.
+ *
+ * @return the protocol scheme for the FileSystem.
+ */
+ public String getScheme() {
+ throw new UnsupportedOperationException("Not implemented by the FileSystem implementation");
+ }
+
/** Returns a URI whose scheme and authority identify this FileSystem.*/
public abstract URI getUri();
@@ -2078,9 +2096,45 @@ public abstract class FileSystem extends
) throws IOException {
}
+ // making it volatile to be able to do a double checked locking
+ private volatile static boolean FILE_SYSTEMS_LOADED = false;
+
+ private static final Map<String, Class<? extends FileSystem>>
+ SERVICE_FILE_SYSTEMS = new HashMap<String, Class<? extends FileSystem>>();
+
+ private static void loadFileSystems() {
+ synchronized (FileSystem.class) {
+ if (!FILE_SYSTEMS_LOADED) {
+ ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class);
+ for (FileSystem fs : serviceLoader) {
+ SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+ }
+ FILE_SYSTEMS_LOADED = true;
+ }
+ }
+ }
+
+ public static Class<? extends FileSystem> getFileSystemClass(String scheme,
+ Configuration conf) throws IOException {
+ if (!FILE_SYSTEMS_LOADED) {
+ loadFileSystems();
+ }
+ Class<? extends FileSystem> clazz = null;
+ if (conf != null) {
+ clazz = (Class<? extends FileSystem>) conf.getClass("fs." + scheme + ".impl", null);
+ }
+ if (clazz == null) {
+ clazz = SERVICE_FILE_SYSTEMS.get(scheme);
+ }
+ if (clazz == null) {
+ throw new IOException("No FileSystem for scheme: " + scheme);
+ }
+ return clazz;
+ }
+
private static FileSystem createFileSystem(URI uri, Configuration conf
) throws IOException {
- Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
+ Class<?> clazz = getFileSystemClass(uri.getScheme(), conf);
if (clazz == null) {
throw new IOException("No FileSystem for scheme: " + uri.getScheme());
}
@@ -2128,8 +2182,8 @@ public abstract class FileSystem extends
}
// now insert the new file system into the map
- if (map.isEmpty() && !clientFinalizer.isAlive()) {
- Runtime.getRuntime().addShutdownHook(clientFinalizer);
+ if (map.isEmpty() ) {
+ ShutdownHookManager.get().addShutdownHook(clientFinalizer, SHUTDOWN_HOOK_PRIORITY);
}
fs.key = key;
map.put(key, fs);
@@ -2144,13 +2198,7 @@ public abstract class FileSystem extends
if (map.containsKey(key) && fs == map.get(key)) {
map.remove(key);
toAutoClose.remove(key);
- if (map.isEmpty() && !clientFinalizer.isAlive()) {
- if (!Runtime.getRuntime().removeShutdownHook(clientFinalizer)) {
- LOG.info("Could not cancel cleanup thread, though no " +
- "FileSystems are open");
- }
}
- }
}
synchronized void closeAll() throws IOException {
@@ -2194,7 +2242,7 @@ public abstract class FileSystem extends
}
}
- private class ClientFinalizer extends Thread {
+ private class ClientFinalizer implements Runnable {
public synchronized void run() {
try {
closeAll(true);
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlStreamHandlerFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlStreamHandlerFactory.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlStreamHandlerFactory.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlStreamHandlerFactory.java Thu May 3 02:14:01 2012
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs;
+import java.io.IOException;
import java.net.URLStreamHandlerFactory;
import java.util.HashMap;
import java.util.Map;
@@ -50,25 +51,23 @@ public class FsUrlStreamHandlerFactory i
private java.net.URLStreamHandler handler;
public FsUrlStreamHandlerFactory() {
- this.conf = new Configuration();
- // force the resolution of the configuration files
- // this is required if we want the factory to be able to handle
- // file:// URLs
- this.conf.getClass("fs.file.impl", null);
- this.handler = new FsUrlStreamHandler(this.conf);
+ this(new Configuration());
}
public FsUrlStreamHandlerFactory(Configuration conf) {
this.conf = new Configuration(conf);
- // force the resolution of the configuration files
- this.conf.getClass("fs.file.impl", null);
this.handler = new FsUrlStreamHandler(this.conf);
}
public java.net.URLStreamHandler createURLStreamHandler(String protocol) {
if (!protocols.containsKey(protocol)) {
- boolean known =
- (conf.getClass("fs." + protocol + ".impl", null) != null);
+ boolean known = true;
+ try {
+ FileSystem.getFileSystemClass(protocol, conf);
+ }
+ catch (IOException ex) {
+ known = false;
+ }
protocols.put(protocol, known);
}
if (protocols.get(protocol)) {
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java Thu May 3 02:14:01 2012
@@ -71,7 +71,18 @@ public class HarFileSystem extends Filte
*/
public HarFileSystem() {
}
-
+
+ /**
+ * Return the protocol scheme for the FileSystem.
+ * <p/>
+ *
+ * @return <code>har</code>
+ */
+ @Override
+ public String getScheme() {
+ return "har";
+ }
+
/**
* Constructor to create a HarFileSystem with an
* underlying filesystem.
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java Thu May 3 02:14:01 2012
@@ -39,7 +39,18 @@ public class LocalFileSystem extends Che
public LocalFileSystem() {
this(new RawLocalFileSystem());
}
-
+
+ /**
+ * Return the protocol scheme for the FileSystem.
+ * <p/>
+ *
+ * @return <code>file</code>
+ */
+ @Override
+ public String getScheme() {
+ return "file";
+ }
+
public FileSystem getRaw() {
return getRawFileSystem();
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java Thu May 3 02:14:01 2012
@@ -59,6 +59,17 @@ public class FTPFileSystem extends FileS
private URI uri;
+ /**
+ * Return the protocol scheme for the FileSystem.
+ * <p/>
+ *
+ * @return <code>ftp</code>
+ */
+ @Override
+ public String getScheme() {
+ return "ftp";
+ }
+
@Override
public void initialize(URI uri, Configuration conf) throws IOException { // get
super.initialize(uri, conf);
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java Thu May 3 02:14:01 2012
@@ -57,6 +57,17 @@ public class KosmosFileSystem extends Fi
this.kfsImpl = fsimpl;
}
+ /**
+ * Return the protocol scheme for the FileSystem.
+ * <p/>
+ *
+ * @return <code>kfs</code>
+ */
+ @Override
+ public String getScheme() {
+ return "kfs";
+ }
+
@Override
public URI getUri() {
return uri;
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java Thu May 3 02:14:01 2012
@@ -67,6 +67,17 @@ public class S3FileSystem extends FileSy
this.store = store;
}
+ /**
+ * Return the protocol scheme for the FileSystem.
+ * <p/>
+ *
+ * @return <code>s3</code>
+ */
+ @Override
+ public String getScheme() {
+ return "s3";
+ }
+
@Override
public URI getUri() {
return uri;
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java Thu May 3 02:14:01 2012
@@ -251,7 +251,18 @@ public class NativeS3FileSystem extends
public NativeS3FileSystem(NativeFileSystemStore store) {
this.store = store;
}
-
+
+ /**
+ * Return the protocol scheme for the FileSystem.
+ * <p/>
+ *
+ * @return <code>s3n</code>
+ */
+ @Override
+ public String getScheme() {
+ return "s3n";
+ }
+
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
super.initialize(uri, conf);
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java Thu May 3 02:14:01 2012
@@ -150,6 +150,17 @@ public class ViewFileSystem extends File
}
/**
+ * Return the protocol scheme for the FileSystem.
+ * <p/>
+ *
+ * @return <code>viewfs</code>
+ */
+ @Override
+ public String getScheme() {
+ return "viewfs";
+ }
+
+ /**
* Called after a new FileSystem instance is constructed.
* @param theUri a uri whose authority section names the host, port, etc. for
* this FileSystem
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FenceMethod.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FenceMethod.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FenceMethod.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FenceMethod.java Thu May 3 02:14:01 2012
@@ -52,7 +52,7 @@ public interface FenceMethod {
/**
* Attempt to fence the target node.
- * @param serviceAddr the address (host:ipcport) of the service to fence
+ * @param target the target of the service to fence
* @param args the configured arguments, which were checked at startup by
* {@link #checkArgs(String)}
* @return true if fencing was successful, false if unsuccessful or
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java Thu May 3 02:14:01 2012
@@ -144,12 +144,13 @@ public interface HAServiceProtocol {
/**
* Return the current status of the service. The status indicates
* the current <em>state</em> (e.g ACTIVE/STANDBY) as well as
- * some additional information. {@see HAServiceStatus}
+ * some additional information.
*
* @throws AccessControlException
* if access is denied.
* @throws IOException
* if other errors happen
+ * @see HAServiceStatus
*/
public HAServiceStatus getServiceStatus() throws AccessControlException,
IOException;
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java Thu May 3 02:14:01 2012
@@ -636,80 +636,16 @@ public class HttpServer implements Filte
*/
public void start() throws IOException {
try {
- if(listenerStartedExternally) { // Expect that listener was started securely
- if(listener.getLocalPort() == -1) // ... and verify
- throw new Exception("Exepected webserver's listener to be started " +
- "previously but wasn't");
- // And skip all the port rolling issues.
+ try {
+ openListener();
+ LOG.info("Jetty bound to port " + listener.getLocalPort());
webServer.start();
- } else {
- int port = 0;
- int oriPort = listener.getPort(); // The original requested port
- while (true) {
- try {
- port = webServer.getConnectors()[0].getLocalPort();
- LOG.debug("Port returned by webServer.getConnectors()[0]." +
- "getLocalPort() before open() is "+ port +
- ". Opening the listener on " + oriPort);
- listener.open();
- port = listener.getLocalPort();
- LOG.debug("listener.getLocalPort() returned " + listener.getLocalPort() +
- " webServer.getConnectors()[0].getLocalPort() returned " +
- webServer.getConnectors()[0].getLocalPort());
- //Workaround to handle the problem reported in HADOOP-4744
- if (port < 0) {
- Thread.sleep(100);
- int numRetries = 1;
- while (port < 0) {
- LOG.warn("listener.getLocalPort returned " + port);
- if (numRetries++ > MAX_RETRIES) {
- throw new Exception(" listener.getLocalPort is returning " +
- "less than 0 even after " +numRetries+" resets");
- }
- for (int i = 0; i < 2; i++) {
- LOG.info("Retrying listener.getLocalPort()");
- port = listener.getLocalPort();
- if (port > 0) {
- break;
- }
- Thread.sleep(200);
- }
- if (port > 0) {
- break;
- }
- LOG.info("Bouncing the listener");
- listener.close();
- Thread.sleep(1000);
- listener.setPort(oriPort == 0 ? 0 : (oriPort += 1));
- listener.open();
- Thread.sleep(100);
- port = listener.getLocalPort();
- }
- } //Workaround end
- LOG.info("Jetty bound to port " + port);
- webServer.start();
- break;
- } catch (IOException ex) {
- // if this is a bind exception,
- // then try the next port number.
- if (ex instanceof BindException) {
- if (!findPort) {
- BindException be = new BindException(
- "Port in use: " + listener.getHost()
- + ":" + listener.getPort());
- be.initCause(ex);
- throw be;
- }
- } else {
- LOG.info("HttpServer.start() threw a non Bind IOException");
- throw ex;
- }
- } catch (MultiException ex) {
- LOG.info("HttpServer.start() threw a MultiException");
- throw ex;
- }
- listener.setPort((oriPort += 1));
- }
+ } catch (IOException ex) {
+ LOG.info("HttpServer.start() threw a non Bind IOException", ex);
+ throw ex;
+ } catch (MultiException ex) {
+ LOG.info("HttpServer.start() threw a MultiException", ex);
+ throw ex;
}
// Make sure there is no handler failures.
Handler[] handlers = webServer.getHandlers();
@@ -730,6 +666,52 @@ public class HttpServer implements Filte
}
/**
+ * Open the main listener for the server
+ * @throws Exception
+ */
+ void openListener() throws Exception {
+ if (listener.getLocalPort() != -1) { // it's already bound
+ return;
+ }
+ if (listenerStartedExternally) { // Expect that listener was started securely
+ throw new Exception("Expected webserver's listener to be started " +
+ "previously but wasn't");
+ }
+ int port = listener.getPort();
+ while (true) {
+ // jetty has a bug where you can't reopen a listener that previously
+ // failed to open w/o issuing a close first, even if the port is changed
+ try {
+ listener.close();
+ listener.open();
+ break;
+ } catch (BindException ex) {
+ if (port == 0 || !findPort) {
+ BindException be = new BindException(
+ "Port in use: " + listener.getHost() + ":" + listener.getPort());
+ be.initCause(ex);
+ throw be;
+ }
+ }
+ // try the next port number
+ listener.setPort(++port);
+ Thread.sleep(100);
+ }
+ }
+
+ /**
+ * Return the bind address of the listener.
+ * @return InetSocketAddress of the listener
+ */
+ public InetSocketAddress getListenerAddress() {
+ int port = listener.getLocalPort();
+ if (port == -1) { // not bound, return requested port
+ port = listener.getPort();
+ }
+ return new InetSocketAddress(listener.getHost(), port);
+ }
+
+ /**
* stop the server
*/
public void stop() throws Exception {
@@ -821,7 +803,10 @@ public class HttpServer implements Filte
String remoteUser = request.getRemoteUser();
if (remoteUser == null) {
- return true;
+ response.sendError(HttpServletResponse.SC_UNAUTHORIZED,
+ "Unauthenticated users are not " +
+ "authorized to access this page.");
+ return false;
}
AccessControlList adminsAcl = (AccessControlList) servletContext
.getAttribute(ADMINS_ACL);
@@ -830,9 +815,7 @@ public class HttpServer implements Filte
if (adminsAcl != null) {
if (!adminsAcl.isUserAllowed(remoteUserUGI)) {
response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "User "
- + remoteUser + " is unauthorized to access this page. "
- + "AccessControlList for accessing this page : "
- + adminsAcl.toString());
+ + remoteUser + " is unauthorized to access this page.");
return false;
}
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java Thu May 3 02:14:01 2012
@@ -1050,9 +1050,9 @@ public class SequenceFile {
int bufferSize = bufferSizeOption == null ? getBufferSize(conf) :
bufferSizeOption.getValue();
short replication = replicationOption == null ?
- fs.getDefaultReplication() :
+ fs.getDefaultReplication(p) :
(short) replicationOption.getValue();
- long blockSize = blockSizeOption == null ? fs.getDefaultBlockSize() :
+ long blockSize = blockSizeOption == null ? fs.getDefaultBlockSize(p) :
blockSizeOption.getValue();
Progressable progress = progressOption == null ? null :
progressOption.getValue();
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java Thu May 3 02:14:01 2012
@@ -326,9 +326,41 @@ public final class WritableUtils {
* @return deserialized integer from stream.
*/
public static int readVInt(DataInput stream) throws IOException {
- return (int) readVLong(stream);
+ long n = readVLong(stream);
+ if ((n > Integer.MAX_VALUE) || (n < Integer.MIN_VALUE)) {
+ throw new IOException("value too long to fit in integer");
+ }
+ return (int)n;
}
-
+
+ /**
+ * Reads an integer from the input stream and returns it.
+ *
+ * This function validates that the integer is between [lower, upper],
+ * inclusive.
+ *
+ * @param stream Binary input stream
+ * @throws java.io.IOException
+ * @return deserialized integer from stream
+ */
+ public static int readVIntInRange(DataInput stream, int lower, int upper)
+ throws IOException {
+ long n = readVLong(stream);
+ if (n < lower) {
+ if (lower == 0) {
+ throw new IOException("expected non-negative integer, got " + n);
+ } else {
+ throw new IOException("expected integer greater than or equal to " +
+ lower + ", got " + n);
+ }
+ }
+ if (n > upper) {
+ throw new IOException("expected integer less or equal to " + upper +
+ ", got " + n);
+ }
+ return (int)n;
+ }
+
/**
* Given the first byte of a vint/vlong, determine the sign
* @param value the first byte
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java Thu May 3 02:14:01 2012
@@ -36,6 +36,9 @@ public class CompressionCodecFactory {
public static final Log LOG =
LogFactory.getLog(CompressionCodecFactory.class.getName());
+
+ private static final ServiceLoader<CompressionCodec> CODEC_PROVIDERS =
+ ServiceLoader.load(CompressionCodec.class);
/**
* A map from the reversed filename suffixes to the codecs.
@@ -95,16 +98,23 @@ public class CompressionCodecFactory {
}
/**
- * Get the list of codecs listed in the configuration
+ * Get the list of codecs discovered via a Java ServiceLoader, or
+ * listed in the configuration. Codecs specified in configuration come
+ * later in the returned list, and are considered to override those
+ * from the ServiceLoader.
* @param conf the configuration to look in
- * @return a list of the Configuration classes or null if the attribute
- * was not set
+ * @return a list of the {@link CompressionCodec} classes
*/
public static List<Class<? extends CompressionCodec>> getCodecClasses(Configuration conf) {
+ List<Class<? extends CompressionCodec>> result
+ = new ArrayList<Class<? extends CompressionCodec>>();
+ // Add codec classes discovered via service loading
+ for (CompressionCodec codec : CODEC_PROVIDERS) {
+ result.add(codec.getClass());
+ }
+ // Add codec classes from configuration
String codecsString = conf.get("io.compression.codecs");
if (codecsString != null) {
- List<Class<? extends CompressionCodec>> result
- = new ArrayList<Class<? extends CompressionCodec>>();
StringTokenizer codecSplit = new StringTokenizer(codecsString, ",");
while (codecSplit.hasMoreElements()) {
String codecSubstring = codecSplit.nextToken();
@@ -123,14 +133,14 @@ public class CompressionCodecFactory {
}
}
}
- return result;
- } else {
- return null;
}
+ return result;
}
/**
- * Sets a list of codec classes in the configuration.
+ * Sets a list of codec classes in the configuration. In addition to any
+ * classes specified using this method, {@link CompressionCodec} classes on
+ * the classpath are discovered using a Java ServiceLoader.
* @param conf the configuration to modify
* @param classes the list of classes to set
*/
@@ -151,21 +161,19 @@ public class CompressionCodecFactory {
/**
* Find the codecs specified in the config value io.compression.codecs
- * and register them. Defaults to gzip and zip.
+ * and register them. Defaults to gzip and deflate.
*/
public CompressionCodecFactory(Configuration conf) {
codecs = new TreeMap<String, CompressionCodec>();
codecsByClassName = new HashMap<String, CompressionCodec>();
codecsByName = new HashMap<String, CompressionCodec>();
List<Class<? extends CompressionCodec>> codecClasses = getCodecClasses(conf);
- if (codecClasses == null) {
+ if (codecClasses == null || codecClasses.isEmpty()) {
addCodec(new GzipCodec());
addCodec(new DefaultCodec());
} else {
- Iterator<Class<? extends CompressionCodec>> itr = codecClasses.iterator();
- while (itr.hasNext()) {
- CompressionCodec codec = ReflectionUtils.newInstance(itr.next(), conf);
- addCodec(codec);
+ for (Class<? extends CompressionCodec> codecClass : codecClasses) {
+ addCodec(ReflectionUtils.newInstance(codecClass, conf));
}
}
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java Thu May 3 02:14:01 2012
@@ -50,8 +50,9 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.ipc.RpcPayloadHeader.*;
import org.apache.hadoop.ipc.protobuf.IpcConnectionContextProtos.IpcConnectionContextProto;
+import org.apache.hadoop.ipc.protobuf.RpcPayloadHeaderProtos.RpcPayloadHeaderProto;
+import org.apache.hadoop.ipc.protobuf.RpcPayloadHeaderProtos.RpcPayloadOperationProto;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
@@ -163,10 +164,10 @@ public class Client {
final Writable rpcRequest; // the serialized rpc request - RpcPayload
Writable rpcResponse; // null if rpc has error
IOException error; // exception, null if success
- final RpcKind rpcKind; // Rpc EngineKind
+ final RPC.RpcKind rpcKind; // Rpc EngineKind
boolean done; // true when call is done
- protected Call(RpcKind rpcKind, Writable param) {
+ protected Call(RPC.RpcKind rpcKind, Writable param) {
this.rpcKind = rpcKind;
this.rpcRequest = param;
synchronized (Client.this) {
@@ -613,7 +614,7 @@ public class Client {
this.in = new DataInputStream(new BufferedInputStream(inStream));
}
this.out = new DataOutputStream(new BufferedOutputStream(outStream));
- writeHeader();
+ writeConnectionContext();
// update last activity time
touch();
@@ -704,16 +705,17 @@ public class Client {
out.flush();
}
- /* Write the protocol header for each connection
+ /* Write the connection context header for each connection
* Out is not synchronized because only the first thread does this.
*/
- private void writeHeader() throws IOException {
+ private void writeConnectionContext() throws IOException {
// Write out the ConnectionHeader
DataOutputBuffer buf = new DataOutputBuffer();
connectionContext.writeTo(buf);
// Write out the payload length
int bufLen = buf.getLength();
+
out.writeInt(bufLen);
out.write(buf.getData(), 0, bufLen);
}
@@ -806,21 +808,22 @@ public class Client {
if (LOG.isDebugEnabled())
LOG.debug(getName() + " sending #" + call.id);
- //for serializing the
- //data to be written
+ // Serializing the data to be written.
+ // Format:
+ // 0) Length of rest below (1 + 2)
+ // 1) PayloadHeader - is serialized Delimited hence contains length
+ // 2) the Payload - the RpcRequest
+ //
d = new DataOutputBuffer();
- d.writeInt(0); // placeholder for data length
- RpcPayloadHeader header = new RpcPayloadHeader(
- call.rpcKind, RpcPayloadOperation.RPC_FINAL_PAYLOAD, call.id);
- header.write(d);
+ RpcPayloadHeaderProto header = ProtoUtil.makeRpcPayloadHeader(
+ call.rpcKind, RpcPayloadOperationProto.RPC_FINAL_PAYLOAD, call.id);
+ header.writeDelimitedTo(d);
call.rpcRequest.write(d);
byte[] data = d.getData();
- int dataLength = d.getLength() - 4;
- data[0] = (byte)((dataLength >>> 24) & 0xff);
- data[1] = (byte)((dataLength >>> 16) & 0xff);
- data[2] = (byte)((dataLength >>> 8) & 0xff);
- data[3] = (byte)(dataLength & 0xff);
- out.write(data, 0, dataLength + 4);//write the data
+
+ int totalLength = d.getLength();
+ out.writeInt(totalLength); // Total Length
+ out.write(data, 0, totalLength);//PayloadHeader + RpcRequest
out.flush();
}
} catch(IOException e) {
@@ -937,7 +940,7 @@ public class Client {
private int index;
public ParallelCall(Writable param, ParallelResults results, int index) {
- super(RpcKind.RPC_WRITABLE, param);
+ super(RPC.RpcKind.RPC_WRITABLE, param);
this.results = results;
this.index = index;
}
@@ -1022,22 +1025,22 @@ public class Client {
}
/**
- * Same as {@link #call(RpcPayloadHeader.RpcKind, Writable, ConnectionId)}
+ * Same as {@link #call(RPC.RpcKind, Writable, ConnectionId)}
* for RPC_BUILTIN
*/
public Writable call(Writable param, InetSocketAddress address)
throws InterruptedException, IOException {
- return call(RpcKind.RPC_BUILTIN, param, address);
+ return call(RPC.RpcKind.RPC_BUILTIN, param, address);
}
/** Make a call, passing <code>param</code>, to the IPC server running at
* <code>address</code>, returning the value. Throws exceptions if there are
* network problems or if the remote code threw an exception.
- * @deprecated Use {@link #call(RpcPayloadHeader.RpcKind, Writable,
+ * @deprecated Use {@link #call(RPC.RpcKind, Writable,
* ConnectionId)} instead
*/
@Deprecated
- public Writable call(RpcKind rpcKind, Writable param, InetSocketAddress address)
+ public Writable call(RPC.RpcKind rpcKind, Writable param, InetSocketAddress address)
throws InterruptedException, IOException {
return call(rpcKind, param, address, null);
}
@@ -1047,11 +1050,11 @@ public class Client {
* the value.
* Throws exceptions if there are network problems or if the remote code
* threw an exception.
- * @deprecated Use {@link #call(RpcPayloadHeader.RpcKind, Writable,
+ * @deprecated Use {@link #call(RPC.RpcKind, Writable,
* ConnectionId)} instead
*/
@Deprecated
- public Writable call(RpcKind rpcKind, Writable param, InetSocketAddress addr,
+ public Writable call(RPC.RpcKind rpcKind, Writable param, InetSocketAddress addr,
UserGroupInformation ticket)
throws InterruptedException, IOException {
ConnectionId remoteId = ConnectionId.getConnectionId(addr, null, ticket, 0,
@@ -1065,11 +1068,11 @@ public class Client {
* timeout, returning the value.
* Throws exceptions if there are network problems or if the remote code
* threw an exception.
- * @deprecated Use {@link #call(RpcPayloadHeader.RpcKind, Writable,
+ * @deprecated Use {@link #call(RPC.RpcKind, Writable,
* ConnectionId)} instead
*/
@Deprecated
- public Writable call(RpcKind rpcKind, Writable param, InetSocketAddress addr,
+ public Writable call(RPC.RpcKind rpcKind, Writable param, InetSocketAddress addr,
Class<?> protocol, UserGroupInformation ticket,
int rpcTimeout)
throws InterruptedException, IOException {
@@ -1080,7 +1083,7 @@ public class Client {
/**
- * Same as {@link #call(RpcPayloadHeader.RpcKind, Writable, InetSocketAddress,
+ * Same as {@link #call(RPC.RpcKind, Writable, InetSocketAddress,
* Class, UserGroupInformation, int, Configuration)}
* except that rpcKind is writable.
*/
@@ -1090,7 +1093,7 @@ public class Client {
throws InterruptedException, IOException {
ConnectionId remoteId = ConnectionId.getConnectionId(addr, protocol,
ticket, rpcTimeout, conf);
- return call(RpcKind.RPC_BUILTIN, param, remoteId);
+ return call(RPC.RpcKind.RPC_BUILTIN, param, remoteId);
}
/**
@@ -1101,7 +1104,7 @@ public class Client {
* value. Throws exceptions if there are network problems or if the remote
* code threw an exception.
*/
- public Writable call(RpcKind rpcKind, Writable param, InetSocketAddress addr,
+ public Writable call(RPC.RpcKind rpcKind, Writable param, InetSocketAddress addr,
Class<?> protocol, UserGroupInformation ticket,
int rpcTimeout, Configuration conf)
throws InterruptedException, IOException {
@@ -1111,12 +1114,12 @@ public class Client {
}
/**
- * Same as {link {@link #call(RpcPayloadHeader.RpcKind, Writable, ConnectionId)}
+ * Same as {link {@link #call(RPC.RpcKind, Writable, ConnectionId)}
* except the rpcKind is RPC_BUILTIN
*/
public Writable call(Writable param, ConnectionId remoteId)
throws InterruptedException, IOException {
- return call(RpcKind.RPC_BUILTIN, param, remoteId);
+ return call(RPC.RpcKind.RPC_BUILTIN, param, remoteId);
}
/**
@@ -1130,7 +1133,7 @@ public class Client {
* Throws exceptions if there are network problems or if the remote code
* threw an exception.
*/
- public Writable call(RpcKind rpcKind, Writable rpcRequest,
+ public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
ConnectionId remoteId) throws InterruptedException, IOException {
Call call = new Call(rpcKind, rpcRequest);
Connection connection = getConnection(remoteId, call);
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java Thu May 3 02:14:01 2012
@@ -38,7 +38,6 @@ import org.apache.hadoop.io.DataOutputOu
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.ipc.RPC.RpcInvoker;
-import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;
import org.apache.hadoop.ipc.protobuf.HadoopRpcProtos.HadoopRpcRequestProto;
import org.apache.hadoop.security.UserGroupInformation;
@@ -61,7 +60,7 @@ public class ProtobufRpcEngine implement
static { // Register the rpcRequest deserializer for WritableRpcEngine
org.apache.hadoop.ipc.Server.registerProtocolEngine(
- RpcKind.RPC_PROTOCOL_BUFFER, RpcRequestWritable.class,
+ RPC.RpcKind.RPC_PROTOCOL_BUFFER, RpcRequestWritable.class,
new Server.ProtoBufRpcInvoker());
}
@@ -182,7 +181,7 @@ public class ProtobufRpcEngine implement
HadoopRpcRequestProto rpcRequest = constructRpcRequest(method, args);
RpcResponseWritable val = null;
try {
- val = (RpcResponseWritable) client.call(RpcKind.RPC_PROTOCOL_BUFFER,
+ val = (RpcResponseWritable) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
new RpcRequestWritable(rpcRequest), remoteId);
} catch (Throwable e) {
throw new ServiceException(e);
@@ -351,7 +350,7 @@ public class ProtobufRpcEngine implement
numReaders, queueSizePerHandler, conf, classNameBase(protocolImpl
.getClass().getName()), secretManager, portRangeConfig);
this.verbose = verbose;
- registerProtocolAndImpl(RpcKind.RPC_PROTOCOL_BUFFER, protocolClass,
+ registerProtocolAndImpl(RPC.RpcKind.RPC_PROTOCOL_BUFFER, protocolClass,
protocolImpl);
}
@@ -363,10 +362,10 @@ public class ProtobufRpcEngine implement
String protoName, long version) throws IOException {
ProtoNameVer pv = new ProtoNameVer(protoName, version);
ProtoClassProtoImpl impl =
- server.getProtocolImplMap(RpcKind.RPC_PROTOCOL_BUFFER).get(pv);
+ server.getProtocolImplMap(RPC.RpcKind.RPC_PROTOCOL_BUFFER).get(pv);
if (impl == null) { // no match for Protocol AND Version
VerProtocolImpl highest =
- server.getHighestSupportedProtocol(RpcKind.RPC_PROTOCOL_BUFFER,
+ server.getHighestSupportedProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
protoName);
if (highest == null) {
throw new IOException("Unknown protocol: " + protoName);
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInfoServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInfoServerSideTranslatorPB.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInfoServerSideTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInfoServerSideTranslatorPB.java Thu May 3 02:14:01 2012
@@ -18,7 +18,6 @@
package org.apache.hadoop.ipc;
import org.apache.hadoop.ipc.RPC.Server.VerProtocolImpl;
-import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;
import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureRequestProto;
import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureResponseProto;
import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolVersionsRequestProto;
@@ -49,7 +48,7 @@ public class ProtocolMetaInfoServerSideT
String protocol = request.getProtocol();
GetProtocolVersionsResponseProto.Builder builder =
GetProtocolVersionsResponseProto.newBuilder();
- for (RpcKind r : RpcKind.values()) {
+ for (RPC.RpcKind r : RPC.RpcKind.values()) {
long[] versions;
try {
versions = getProtocolVersionForRpcKind(r, protocol);
@@ -78,7 +77,7 @@ public class ProtocolMetaInfoServerSideT
String rpcKind = request.getRpcKind();
long[] versions;
try {
- versions = getProtocolVersionForRpcKind(RpcKind.valueOf(rpcKind),
+ versions = getProtocolVersionForRpcKind(RPC.RpcKind.valueOf(rpcKind),
protocol);
} catch (ClassNotFoundException e1) {
throw new ServiceException(e1);
@@ -104,7 +103,7 @@ public class ProtocolMetaInfoServerSideT
return builder.build();
}
- private long[] getProtocolVersionForRpcKind(RpcKind rpcKind,
+ private long[] getProtocolVersionForRpcKind(RPC.RpcKind rpcKind,
String protocol) throws ClassNotFoundException {
Class<?> protocolClass = Class.forName(protocol);
String protocolName = RPC.getProtocolName(protocolClass);