You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2012/05/12 22:52:42 UTC
svn commit: r1337645 [1/2] - in
/hadoop/common/branches/HDFS-3042/hadoop-common-project:
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/
hadoop-auth...
Author: todd
Date: Sat May 12 20:52:34 2012
New Revision: 1337645
URL: http://svn.apache.org/viewvc?rev=1337645&view=rev
Log:
Merge trunk into auto-HA branch
Added:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputWrapper.java
- copied unchanged from r1337618, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputWrapper.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/resources/META-INF/
- copied from r1337618, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/META-INF/
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/
- copied from r1337618, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
- copied unchanged from r1337618, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
Removed:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Status.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Krb5AndCertsSslSocketConnector.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestBlockLocation.java
Modified:
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionUtil.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/configure.ac
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/log4j.properties
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/proto/RpcPayloadHeader.proto
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSocketIOWithTimeout.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestToken.java
hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestVersionUtil.java
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java Sat May 12 20:52:34 2012
@@ -26,7 +26,6 @@ import javax.security.auth.login.Configu
import javax.security.auth.login.LoginContext;
import javax.security.auth.login.LoginException;
import java.io.IOException;
-import java.lang.reflect.Field;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.AccessControlContext;
@@ -196,11 +195,10 @@ public class KerberosAuthenticator imple
try {
GSSManager gssManager = GSSManager.getInstance();
String servicePrincipal = "HTTP/" + KerberosAuthenticator.this.url.getHost();
-
+ Oid oid = KerberosUtil.getOidInstance("NT_GSS_KRB5_PRINCIPAL");
GSSName serviceName = gssManager.createName(servicePrincipal,
- GSSName.NT_HOSTBASED_SERVICE);
- Oid oid = KerberosUtil.getOidClassInstance(servicePrincipal,
- gssManager);
+ oid);
+ oid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID");
gssContext = gssManager.createContext(serviceName, oid, null,
GSSContext.DEFAULT_LIFETIME);
gssContext.requestCredDeleg(true);
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java Sat May 12 20:52:34 2012
@@ -327,6 +327,8 @@ public class AuthenticationFilter implem
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain)
throws IOException, ServletException {
+ boolean unauthorizedResponse = true;
+ String unauthorizedMsg = "";
HttpServletRequest httpRequest = (HttpServletRequest) request;
HttpServletResponse httpResponse = (HttpServletResponse) response;
try {
@@ -350,6 +352,7 @@ public class AuthenticationFilter implem
newToken = true;
}
if (token != null) {
+ unauthorizedResponse = false;
if (LOG.isDebugEnabled()) {
LOG.debug("Request [{}] user [{}] authenticated", getRequestURL(httpRequest), token.getUserName());
}
@@ -378,17 +381,17 @@ public class AuthenticationFilter implem
}
filterChain.doFilter(httpRequest, httpResponse);
}
- else {
- throw new AuthenticationException("Missing AuthenticationToken");
- }
} catch (AuthenticationException ex) {
+ unauthorizedMsg = ex.toString();
+ LOG.warn("Authentication exception: " + ex.getMessage(), ex);
+ }
+ if (unauthorizedResponse) {
if (!httpResponse.isCommitted()) {
Cookie cookie = createCookie("");
cookie.setMaxAge(0);
httpResponse.addCookie(cookie);
- httpResponse.sendError(HttpServletResponse.SC_UNAUTHORIZED, ex.getMessage());
+ httpResponse.sendError(HttpServletResponse.SC_UNAUTHORIZED, unauthorizedMsg);
}
- LOG.warn("Authentication exception: " + ex.getMessage(), ex);
}
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java Sat May 12 20:52:34 2012
@@ -22,7 +22,6 @@ import java.lang.reflect.InvocationTarge
import java.lang.reflect.Method;
import org.ietf.jgss.GSSException;
-import org.ietf.jgss.GSSManager;
import org.ietf.jgss.Oid;
public class KerberosUtil {
@@ -34,8 +33,7 @@ public class KerberosUtil {
: "com.sun.security.auth.module.Krb5LoginModule";
}
- public static Oid getOidClassInstance(String servicePrincipal,
- GSSManager gssManager)
+ public static Oid getOidInstance(String oidName)
throws ClassNotFoundException, GSSException, NoSuchFieldException,
IllegalAccessException {
Class<?> oidClass;
@@ -44,7 +42,7 @@ public class KerberosUtil {
} else {
oidClass = Class.forName("sun.security.jgss.GSSUtil");
}
- Field oidField = oidClass.getDeclaredField("GSS_KRB5_MECH_OID");
+ Field oidField = oidClass.getDeclaredField(oidName);
return (Oid)oidField.get(oidClass);
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java Sat May 12 20:52:34 2012
@@ -145,10 +145,10 @@ public class TestKerberosAuthenticationH
GSSContext gssContext = null;
try {
String servicePrincipal = KerberosTestUtils.getServerPrincipal();
+ Oid oid = KerberosUtil.getOidInstance("NT_GSS_KRB5_PRINCIPAL");
GSSName serviceName = gssManager.createName(servicePrincipal,
- GSSName.NT_HOSTBASED_SERVICE);
- Oid oid = KerberosUtil.getOidClassInstance(servicePrincipal,
- gssManager);
+ oid);
+ oid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID");
gssContext = gssManager.createContext(serviceName, oid, null,
GSSContext.DEFAULT_LIFETIME);
gssContext.requestCredDeleg(true);
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt Sat May 12 20:52:34 2012
@@ -63,8 +63,6 @@ Trunk (unreleased changes)
HADOOP-8290. Remove remaining references to hadoop.native.lib (harsh)
- HADOOP-8285 Use ProtoBuf for RpcPayLoadHeader (sanjay radia)
-
HADOOP-8308. Support cross-project Jenkins builds. (tomwhite)
BUG FIXES
@@ -129,6 +127,15 @@ Trunk (unreleased changes)
HADOOP-8339. jenkins complaining about 16 javadoc warnings
(Tom White and Robert Evans via tgraves)
+ HADOOP-8354. test-patch findbugs may fail if a dependent module is changed
+ (Tom White and Robert Evans)
+
+ HADOOP-8375. test-patch should stop immediately once it has found
+ compilation errors (bobby)
+
+ HADOOP-8395. Text shell command unnecessarily demands that a
+ SequenceFile's key class be WritableComparable (harsh)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -139,6 +146,9 @@ Release 2.0.0 - UNRELEASED
HADOOP-7920. Remove Avro Rpc. (suresh)
+ HADOOP-8388. Remove unused BlockLocation serialization.
+ (Colin Patrick McCabe via eli)
+
NEW FEATURES
HADOOP-7773. Add support for protocol buffer based RPC engine.
@@ -163,6 +173,9 @@ Release 2.0.0 - UNRELEASED
HADOOP-8210. Common side of HDFS-3148: The client should be able
to use multiple local interfaces for data transfer. (eli)
+ HADOOP-8343. Allow configuration of authorization for JmxJsonServlet and
+ MetricsServlet (tucu)
+
IMPROVEMENTS
HADOOP-7524. Change RPC to allow multiple protocols including multuple
@@ -284,6 +297,34 @@ Release 2.0.0 - UNRELEASED
HADOOP-8214. make hadoop script recognize a full set of deprecated commands (rvs via tucu)
+ HADOOP-8347. Hadoop Common logs misspell 'successful'.
+ (Philip Zeyliger via eli)
+
+ HADOOP-8350. Improve NetUtils.getInputStream to return a stream which has
+ a tunable timeout. (todd)
+
+ HADOOP-8356. FileSystem service loading mechanism should print the FileSystem
+ impl it is failing to load (tucu)
+
+ HADOOP-8340. SNAPSHOT build versions should compare as less than their eventual
+ final release. (todd)
+
+ HADOOP-8361. Avoid out-of-memory problems when deserializing strings.
+ (Colin Patrick McCabe via eli)
+
+ HADOOP-8353. hadoop-daemon.sh and yarn-daemon.sh can be misleading on stop.
+ (Roman Shaposhnik via atm)
+
+ HADOOP-8224. Don't hardcode hdfs.audit.logger in the scripts.
+ (Tomohiko Kinebuchi via eli)
+
+ HADOOP-8113. Correction to BUILDING.txt: HDFS needs ProtocolBuffer, too
+ (not just MapReduce). Contributed by Eugene Koontz.
+
+ HADOOP-8285 Use ProtoBuf for RpcPayLoadHeader (sanjay radia)
+
+ HADOOP-8366 Use ProtoBuf for RpcResponseHeader (sanjay radia)
+
OPTIMIZATIONS
BUG FIXES
@@ -314,8 +355,6 @@ Release 2.0.0 - UNRELEASED
HADOOP-8104. Inconsistent Jackson versions (tucu)
- HADOOP-7940. The Text.clear() method does not clear the bytes as intended. (Csaba Miklos via harsh)
-
HADOOP-8119. Fix javac warnings in TestAuthenticationFilter in hadoop-auth.
(szetszwo)
@@ -406,6 +445,22 @@ Release 2.0.0 - UNRELEASED
HADOOP-8342. HDFS command fails with exception following merge of
HADOOP-8325 (tucu)
+ HADOOP-8346. Makes oid changes to make SPNEGO work. Was broken due
+ to fixes introduced by the IBM JDK compatibility patch. (ddas)
+
+ HADOOP-8355. SPNEGO filter throws/logs exception when authentication fails (tucu)
+
+ HADOOP-8349. ViewFS doesn't work when the root of a file system is mounted. (atm)
+
+ HADOOP-8328. Duplicate FileSystem Statistics object for 'file' scheme.
+ (tomwhite)
+
+ HADOOP-8359. Fix javadoc warnings in Configuration. (Anupam Seth via
+ szetszwo)
+
+ HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname
+ starting with a numeric character. (Junping Du via suresh)
+
BREAKDOWN OF HADOOP-7454 SUBTASKS
HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh)
@@ -464,6 +519,11 @@ Release 2.0.0 - UNRELEASED
HADOOP-8172. Configuration no longer sets all keys in a deprecated key
list. (Anupam Seth via bobby)
+ HADOOP-7868. Hadoop native fails to compile when default linker
+ option is -Wl,--as-needed. (Trevor Robinson via eli)
+
+ HADOOP-8316. Audit logging should be disabled by default. (eli)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -519,6 +579,13 @@ Release 0.23.3 - UNRELEASED
HADOOP-8335. Improve Configuration's address handling (Daryn Sharp via
bobby)
+ HADOOP-8327. distcpv2 and distcpv1 jars should not coexist (Dave Thompson
+ via bobby)
+
+ HADOOP-8341. Fix or filter findbugs issues in hadoop-tools (bobby)
+
+ HADOOP-8373. Port RPC.getServerAddress to 0.23 (Daryn Sharp via bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1333291-1337618
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh Sat May 12 20:52:34 2012
@@ -109,8 +109,10 @@ fi
export HADOOP_LOGFILE=hadoop-$HADOOP_IDENT_STRING-$command-$HOSTNAME.log
export HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-"INFO,RFA"}
export HADOOP_SECURITY_LOGGER=${HADOOP_SECURITY_LOGGER:-"INFO,RFAS"}
+export HDFS_AUDIT_LOGGER=${HDFS_AUDIT_LOGGER:-"INFO,NullAppender"}
log=$HADOOP_LOG_DIR/hadoop-$HADOOP_IDENT_STRING-$command-$HOSTNAME.out
pid=$HADOOP_PID_DIR/hadoop-$HADOOP_IDENT_STRING-$command.pid
+HADOOP_STOP_TIMEOUT=${HADOOP_STOP_TIMEOUT:-5}
# Set default scheduling priority
if [ "$HADOOP_NICENESS" = "" ]; then
@@ -162,9 +164,15 @@ case $startStop in
(stop)
if [ -f $pid ]; then
- if kill -0 `cat $pid` > /dev/null 2>&1; then
+ TARGET_PID=`cat $pid`
+ if kill -0 $TARGET_PID > /dev/null 2>&1; then
echo stopping $command
- kill `cat $pid`
+ kill $TARGET_PID
+ sleep $HADOOP_STOP_TIMEOUT
+ if kill -0 $TARGET_PID > /dev/null 2>&1; then
+ echo "$command did not stop gracefully after $HADOOP_STOP_TIMEOUT seconds: killing with kill -9"
+ kill -9 $TARGET_PID
+ fi
else
echo no $command to stop
fi
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties Sat May 12 20:52:34 2012
@@ -102,7 +102,7 @@ log4j.appender.TLA.layout.ConversionPatt
#
#Security appender
#
-hadoop.security.logger=INFO,console
+hadoop.security.logger=INFO,NullAppender
hadoop.security.log.maxfilesize=256MB
hadoop.security.log.maxbackupindex=20
log4j.category.SecurityLogger=${hadoop.security.logger}
@@ -126,7 +126,7 @@ log4j.appender.DRFAS.DatePattern=.yyyy-M
#
# hdfs audit logging
#
-hdfs.audit.logger=INFO,console
+hdfs.audit.logger=INFO,NullAppender
hdfs.audit.log.maxfilesize=256MB
hdfs.audit.log.maxbackupindex=20
log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
@@ -141,7 +141,7 @@ log4j.appender.RFAAUDIT.MaxBackupIndex=$
#
# mapred audit logging
#
-mapred.audit.logger=INFO,console
+mapred.audit.logger=INFO,NullAppender
mapred.audit.log.maxfilesize=256MB
mapred.audit.log.maxbackupindex=20
log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
Propchange: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1333291-1337618
Propchange: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1333291-1337618
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java Sat May 12 20:52:34 2012
@@ -18,7 +18,6 @@
package org.apache.hadoop.conf;
import java.io.IOException;
-import java.io.OutputStreamWriter;
import java.io.Writer;
import javax.servlet.ServletException;
@@ -57,9 +56,8 @@ public class ConfServlet extends HttpSer
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
- // Do the authorization
- if (!HttpServer.hasAdministratorAccess(getServletContext(), request,
- response)) {
+ if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(),
+ request, response)) {
return;
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java Sat May 12 20:52:34 2012
@@ -278,7 +278,7 @@ public class Configuration implements It
* @param key
* @param newKeys
* @param customMessage
- * @deprecated use {@link addDeprecation(String key, String newKey,
+ * @deprecated use {@link #addDeprecation(String key, String newKey,
String customMessage)} instead
*/
@Deprecated
@@ -328,7 +328,7 @@ public class Configuration implements It
*
* @param key Key that is to be deprecated
* @param newKeys list of keys that take up the values of deprecated key
- * @deprecated use {@link addDeprecation(String key, String newKey)} instead
+ * @deprecated use {@link #addDeprecation(String key, String newKey)} instead
*/
@Deprecated
public synchronized static void addDeprecation(String key, String[] newKeys) {
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java Sat May 12 20:52:34 2012
@@ -346,7 +346,7 @@ public abstract class AbstractFileSystem
path);
} else {
throw new InvalidPathException(
- "Path without scheme with non-null autorhrity:" + path);
+ "Path without scheme with non-null authority:" + path);
}
}
String thisScheme = this.getUri().getScheme();
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java Sat May 12 20:52:34 2012
@@ -35,16 +35,7 @@ import org.apache.hadoop.io.WritableFact
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class BlockLocation implements Writable {
-
- static { // register a ctor
- WritableFactories.setFactory
- (BlockLocation.class,
- new WritableFactory() {
- public Writable newInstance() { return new BlockLocation(); }
- });
- }
-
+public class BlockLocation {
private String[] hosts; //hostnames of datanodes
private String[] names; //hostname:portNumber of datanodes
private String[] topologyPaths; // full path name in network topology
@@ -219,62 +210,6 @@ public class BlockLocation implements Wr
}
}
- /**
- * Implement write of Writable
- */
- public void write(DataOutput out) throws IOException {
- out.writeLong(offset);
- out.writeLong(length);
- out.writeBoolean(corrupt);
- out.writeInt(names.length);
- for (int i=0; i < names.length; i++) {
- Text name = new Text(names[i]);
- name.write(out);
- }
- out.writeInt(hosts.length);
- for (int i=0; i < hosts.length; i++) {
- Text host = new Text(hosts[i]);
- host.write(out);
- }
- out.writeInt(topologyPaths.length);
- for (int i=0; i < topologyPaths.length; i++) {
- Text host = new Text(topologyPaths[i]);
- host.write(out);
- }
- }
-
- /**
- * Implement readFields of Writable
- */
- public void readFields(DataInput in) throws IOException {
- this.offset = in.readLong();
- this.length = in.readLong();
- this.corrupt = in.readBoolean();
- int numNames = in.readInt();
- this.names = new String[numNames];
- for (int i = 0; i < numNames; i++) {
- Text name = new Text();
- name.readFields(in);
- names[i] = name.toString();
- }
-
- int numHosts = in.readInt();
- this.hosts = new String[numHosts];
- for (int i = 0; i < numHosts; i++) {
- Text host = new Text();
- host.readFields(in);
- hosts[i] = host.toString();
- }
-
- int numTops = in.readInt();
- topologyPaths = new String[numTops];
- for (int i = 0; i < numTops; i++) {
- Text path = new Text();
- path.readFields(in);
- topologyPaths[i] = path.toString();
- }
- }
-
public String toString() {
StringBuilder result = new StringBuilder();
result.append(offset);
@@ -289,4 +224,4 @@ public class BlockLocation implements Wr
}
return result.toString();
}
-}
+}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java Sat May 12 20:52:34 2012
@@ -228,6 +228,9 @@ public class CommonConfigurationKeysPubl
public static final String HADOOP_SECURITY_AUTHORIZATION =
"hadoop.security.authorization";
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN =
+ "hadoop.security.instrumentation.requires.admin";
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String HADOOP_SECURITY_SERVICE_USER_NAME_KEY =
"hadoop.security.service.user.name.key";
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java Sat May 12 20:52:34 2012
@@ -254,7 +254,7 @@ public class FileStatus implements Writa
// Writable
//////////////////////////////////////////////////
public void write(DataOutput out) throws IOException {
- Text.writeString(out, getPath().toString());
+ Text.writeString(out, getPath().toString(), Text.ONE_MEGABYTE);
out.writeLong(getLen());
out.writeBoolean(isDirectory());
out.writeShort(getReplication());
@@ -262,16 +262,16 @@ public class FileStatus implements Writa
out.writeLong(getModificationTime());
out.writeLong(getAccessTime());
getPermission().write(out);
- Text.writeString(out, getOwner());
- Text.writeString(out, getGroup());
+ Text.writeString(out, getOwner(), Text.ONE_MEGABYTE);
+ Text.writeString(out, getGroup(), Text.ONE_MEGABYTE);
out.writeBoolean(isSymlink());
if (isSymlink()) {
- Text.writeString(out, getSymlink().toString());
+ Text.writeString(out, getSymlink().toString(), Text.ONE_MEGABYTE);
}
}
public void readFields(DataInput in) throws IOException {
- String strPath = Text.readString(in);
+ String strPath = Text.readString(in, Text.ONE_MEGABYTE);
this.path = new Path(strPath);
this.length = in.readLong();
this.isdir = in.readBoolean();
@@ -280,10 +280,10 @@ public class FileStatus implements Writa
modification_time = in.readLong();
access_time = in.readLong();
permission.readFields(in);
- owner = Text.readString(in);
- group = Text.readString(in);
+ owner = Text.readString(in, Text.ONE_MEGABYTE);
+ group = Text.readString(in, Text.ONE_MEGABYTE);
if (in.readBoolean()) {
- this.symlink = new Path(Text.readString(in));
+ this.symlink = new Path(Text.readString(in, Text.ONE_MEGABYTE));
} else {
this.symlink = null;
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Sat May 12 20:52:34 2012
@@ -199,7 +199,7 @@ public abstract class FileSystem extends
* @return the protocol scheme for the FileSystem.
*/
public String getScheme() {
- throw new UnsupportedOperationException("Not implemented by the FileSystem implementation");
+ throw new UnsupportedOperationException("Not implemented by the " + getClass().getSimpleName() + " FileSystem implementation");
}
/** Returns a URI whose scheme and authority identify this FileSystem.*/
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java Sat May 12 20:52:34 2012
@@ -53,7 +53,7 @@ import org.apache.hadoop.util.Progressab
public class FilterFileSystem extends FileSystem {
protected FileSystem fs;
- private String swapScheme;
+ protected String swapScheme;
/*
* so that extending classes can define it
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java Sat May 12 20:52:34 2012
@@ -39,6 +39,17 @@ public class LocalFileSystem extends Che
public LocalFileSystem() {
this(new RawLocalFileSystem());
}
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ if (fs.getConf() == null) {
+ fs.initialize(name, conf);
+ }
+ String scheme = name.getScheme();
+ if (!scheme.equals(fs.getUri().getScheme())) {
+ swapScheme = scheme;
+ }
+ }
/**
* Return the protocol scheme for the FileSystem.
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java Sat May 12 20:52:34 2012
@@ -223,6 +223,13 @@ public class Path implements Comparable
return isUriPathAbsolute();
}
+ /**
+ * @return true if and only if this path represents the root of a file system
+ */
+ public boolean isRoot() {
+ return getParent() == null;
+ }
+
/** Returns the final component of this path.*/
public String getName() {
String path = uri.getPath();
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java Sat May 12 20:52:34 2012
@@ -84,8 +84,8 @@ public class PermissionStatus implements
/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
- username = Text.readString(in);
- groupname = Text.readString(in);
+ username = Text.readString(in, Text.ONE_MEGABYTE);
+ groupname = Text.readString(in, Text.ONE_MEGABYTE);
permission = FsPermission.read(in);
}
@@ -110,8 +110,8 @@ public class PermissionStatus implements
String username,
String groupname,
FsPermission permission) throws IOException {
- Text.writeString(out, username);
- Text.writeString(out, groupname);
+ Text.writeString(out, username, Text.ONE_MEGABYTE);
+ Text.writeString(out, groupname, Text.ONE_MEGABYTE);
permission.write(out);
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java Sat May 12 20:52:34 2012
@@ -34,7 +34,6 @@ import org.apache.hadoop.io.DataOutputBu
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.util.ReflectionUtils;
@@ -136,7 +135,7 @@ class Display extends FsCommand {
protected class TextRecordInputStream extends InputStream {
SequenceFile.Reader r;
- WritableComparable<?> key;
+ Writable key;
Writable val;
DataInputBuffer inbuf;
@@ -148,7 +147,7 @@ class Display extends FsCommand {
r = new SequenceFile.Reader(lconf,
SequenceFile.Reader.file(fpath));
key = ReflectionUtils.newInstance(
- r.getKeyClass().asSubclass(WritableComparable.class), lconf);
+ r.getKeyClass().asSubclass(Writable.class), lconf);
val = ReflectionUtils.newInstance(
r.getValueClass().asSubclass(Writable.class), lconf);
inbuf = new DataInputBuffer();
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java Sat May 12 20:52:34 2012
@@ -75,7 +75,8 @@ class ChRootedFileSystem extends FilterF
protected Path fullPath(final Path path) {
super.checkPath(path);
return path.isAbsolute() ?
- new Path(chRootPathPartString + path.toUri().getPath()) :
+ new Path((chRootPathPart.isRoot() ? "" : chRootPathPartString)
+ + path.toUri().getPath()) :
new Path(chRootPathPartString + workingDir.toUri().getPath(), path);
}
@@ -127,7 +128,7 @@ class ChRootedFileSystem extends FilterF
}
String pathPart = p.toUri().getPath();
return (pathPart.length() == chRootPathPartString.length()) ? "" : pathPart
- .substring(chRootPathPartString.length() + 1);
+ .substring(chRootPathPartString.length() + (chRootPathPart.isRoot() ? 0 : 1));
}
@Override
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java Sat May 12 20:52:34 2012
@@ -79,7 +79,8 @@ class ChRootedFs extends AbstractFileSys
*/
protected Path fullPath(final Path path) {
super.checkPath(path);
- return new Path(chRootPathPartString + path.toUri().getPath());
+ return new Path((chRootPathPart.isRoot() ? "" : chRootPathPartString)
+ + path.toUri().getPath());
}
public ChRootedFs(final AbstractFileSystem fs, final Path theRoot)
@@ -127,7 +128,8 @@ class ChRootedFs extends AbstractFileSys
}
String pathPart = p.toUri().getPath();
return (pathPart.length() == chRootPathPartString.length()) ?
- "" : pathPart.substring(chRootPathPartString.length() + 1);
+ "" : pathPart.substring(chRootPathPartString.length() +
+ (chRootPathPart.isRoot() ? 0 : 1));
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java Sat May 12 20:52:34 2012
@@ -52,8 +52,6 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.jmx.JMXJsonServlet;
import org.apache.hadoop.log.LogLevel;
import org.apache.hadoop.metrics.MetricsServlet;
-import org.apache.hadoop.security.Krb5AndCertsSslSocketConnector;
-import org.apache.hadoop.security.Krb5AndCertsSslSocketConnector.MODE;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.util.ReflectionUtils;
@@ -99,6 +97,7 @@ public class HttpServer implements Filte
// gets stored.
public static final String CONF_CONTEXT_ATTRIBUTE = "hadoop.conf";
static final String ADMINS_ACL = "admins.acl";
+ public static final String SPNEGO_FILTER = "SpnegoFilter";
public static final String BIND_ADDRESS = "bind.address";
@@ -237,11 +236,7 @@ public class HttpServer implements Filte
webServer.addHandler(webAppContext);
addDefaultApps(contexts, appDir, conf);
-
- defineFilter(webAppContext, "krb5Filter",
- Krb5AndCertsSslSocketConnector.Krb5SslFilter.class.getName(),
- null, null);
-
+
addGlobalFilter("safety", QuotingInputFilter.class.getName(), null);
final FilterInitializer[] initializers = getFilterInitializers(conf);
if (initializers != null) {
@@ -424,12 +419,13 @@ public class HttpServer implements Filte
* protect with Kerberos authentication.
* Note: This method is to be used for adding servlets that facilitate
* internal communication and not for user facing functionality. For
- * servlets added using this method, filters (except internal Kerberized
+ + * servlets added using this method, filters (except internal Kerberos
* filters) are not enabled.
*
* @param name The name of the servlet (can be passed as null)
* @param pathSpec The path spec for the servlet
* @param clazz The servlet class
+ * @param requireAuth Require Kerberos authenticate to access servlet
*/
public void addInternalServlet(String name, String pathSpec,
Class<? extends HttpServlet> clazz, boolean requireAuth) {
@@ -440,11 +436,11 @@ public class HttpServer implements Filte
webAppContext.addServlet(holder, pathSpec);
if(requireAuth && UserGroupInformation.isSecurityEnabled()) {
- LOG.info("Adding Kerberos filter to " + name);
+ LOG.info("Adding Kerberos (SPNEGO) filter to " + name);
ServletHandler handler = webAppContext.getServletHandler();
FilterMapping fmap = new FilterMapping();
fmap.setPathSpec(pathSpec);
- fmap.setFilterName("krb5Filter");
+ fmap.setFilterName(SPNEGO_FILTER);
fmap.setDispatches(Handler.ALL);
handler.addFilterMapping(fmap);
}
@@ -584,22 +580,10 @@ public class HttpServer implements Filte
* Configure an ssl listener on the server.
* @param addr address to listen on
* @param sslConf conf to retrieve ssl options
- * @param needClientAuth whether client authentication is required
- */
- public void addSslListener(InetSocketAddress addr, Configuration sslConf,
- boolean needClientAuth) throws IOException {
- addSslListener(addr, sslConf, needClientAuth, false);
- }
-
- /**
- * Configure an ssl listener on the server.
- * @param addr address to listen on
- * @param sslConf conf to retrieve ssl options
* @param needCertsAuth whether x509 certificate authentication is required
- * @param needKrbAuth whether to allow kerberos auth
*/
public void addSslListener(InetSocketAddress addr, Configuration sslConf,
- boolean needCertsAuth, boolean needKrbAuth) throws IOException {
+ boolean needCertsAuth) throws IOException {
if (webServer.isStarted()) {
throw new IOException("Failed to add ssl listener");
}
@@ -612,15 +596,7 @@ public class HttpServer implements Filte
System.setProperty("javax.net.ssl.trustStoreType", sslConf.get(
"ssl.server.truststore.type", "jks"));
}
- Krb5AndCertsSslSocketConnector.MODE mode;
- if(needCertsAuth && needKrbAuth)
- mode = MODE.BOTH;
- else if (!needCertsAuth && needKrbAuth)
- mode = MODE.KRB;
- else // Default to certificates
- mode = MODE.CERTS;
-
- SslSocketConnector sslListener = new Krb5AndCertsSslSocketConnector(mode);
+ SslSocketConnector sslListener = new SslSocketConnector();
sslListener.setHost(addr.getHostName());
sslListener.setPort(addr.getPort());
sslListener.setKeystore(sslConf.get("ssl.server.keystore.location"));
@@ -780,6 +756,37 @@ public class HttpServer implements Filte
}
/**
+ * Checks the user has privileges to access to instrumentation servlets.
+ * <p/>
+ * If <code>hadoop.security.instrumentation.requires.admin</code> is set to FALSE
+ * (default value) it always returns TRUE.
+ * <p/>
+ * If <code>hadoop.security.instrumentation.requires.admin</code> is set to TRUE
+ * it will check that if the current user is in the admin ACLS. If the user is
+ * in the admin ACLs it returns TRUE, otherwise it returns FALSE.
+ *
+ * @param servletContext the servlet context.
+ * @param request the servlet request.
+ * @param response the servlet response.
+ * @return TRUE/FALSE based on the logic decribed above.
+ */
+ public static boolean isInstrumentationAccessAllowed(
+ ServletContext servletContext, HttpServletRequest request,
+ HttpServletResponse response) throws IOException {
+ Configuration conf =
+ (Configuration) servletContext.getAttribute(CONF_CONTEXT_ATTRIBUTE);
+
+ boolean access = true;
+ boolean adminAccess = conf.getBoolean(
+ CommonConfigurationKeys.HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN,
+ false);
+ if (adminAccess) {
+ access = hasAdministratorAccess(servletContext, request, response);
+ }
+ return access;
+ }
+
+ /**
* Does the user sending the HttpServletRequest has the administrator ACLs? If
* it isn't the case, response will be modified to send an error to the user.
*
@@ -794,7 +801,6 @@ public class HttpServer implements Filte
HttpServletResponse response) throws IOException {
Configuration conf =
(Configuration) servletContext.getAttribute(CONF_CONTEXT_ATTRIBUTE);
-
// If there is no authorization, anybody has administrator access.
if (!conf.getBoolean(
CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false)) {
@@ -834,12 +840,11 @@ public class HttpServer implements Filte
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
- response.setContentType("text/plain; charset=UTF-8");
- // Do the authorization
- if (!HttpServer.hasAdministratorAccess(getServletContext(), request,
- response)) {
+ if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(),
+ request, response)) {
return;
}
+ response.setContentType("text/plain; charset=UTF-8");
PrintWriter out = response.getWriter();
ReflectionUtils.printThreadInfo(out, "");
out.close();
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java Sat May 12 20:52:34 2012
@@ -239,7 +239,6 @@ public class Text extends BinaryComparab
*/
public void clear() {
length = 0;
- bytes = EMPTY_BYTES;
}
/*
@@ -413,6 +412,8 @@ public class Text extends BinaryComparab
return bytes;
}
+ static final public int ONE_MEGABYTE = 1024 * 1024;
+
/** Read a UTF8 encoded string from in
*/
public static String readString(DataInput in) throws IOException {
@@ -421,7 +422,17 @@ public class Text extends BinaryComparab
in.readFully(bytes, 0, length);
return decode(bytes);
}
-
+
+ /** Read a UTF8 encoded string with a maximum size
+ */
+ public static String readString(DataInput in, int maxLength)
+ throws IOException {
+ int length = WritableUtils.readVIntInRange(in, 0, maxLength - 1);
+ byte [] bytes = new byte[length];
+ in.readFully(bytes, 0, length);
+ return decode(bytes);
+ }
+
/** Write a UTF8 encoded string to out
*/
public static int writeString(DataOutput out, String s) throws IOException {
@@ -432,6 +443,22 @@ public class Text extends BinaryComparab
return length;
}
+ /** Write a UTF8 encoded string with a maximum size to out
+ */
+ public static int writeString(DataOutput out, String s, int maxLength)
+ throws IOException {
+ ByteBuffer bytes = encode(s);
+ int length = bytes.limit();
+ if (length >= maxLength) {
+ throw new IOException("string was too long to write! Expected " +
+ "less than " + maxLength + " bytes, but got " +
+ length + " bytes.");
+ }
+ WritableUtils.writeVInt(out, length);
+ out.write(bytes.array(), 0, length);
+ return length;
+ }
+
////// states for validateUTF8
private static final int LEAD_BYTE = 0;
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java Sat May 12 20:52:34 2012
@@ -53,6 +53,8 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.ipc.protobuf.IpcConnectionContextProtos.IpcConnectionContextProto;
import org.apache.hadoop.ipc.protobuf.RpcPayloadHeaderProtos.RpcPayloadHeaderProto;
import org.apache.hadoop.ipc.protobuf.RpcPayloadHeaderProtos.RpcPayloadOperationProto;
+import org.apache.hadoop.ipc.protobuf.RpcPayloadHeaderProtos.RpcResponseHeaderProto;
+import org.apache.hadoop.ipc.protobuf.RpcPayloadHeaderProtos.RpcStatusProto;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
@@ -845,24 +847,24 @@ public class Client {
touch();
try {
- int id = in.readInt(); // try to read an id
-
+ RpcResponseHeaderProto response =
+ RpcResponseHeaderProto.parseDelimitedFrom(in);
+ int callId = response.getCallId();
if (LOG.isDebugEnabled())
- LOG.debug(getName() + " got value #" + id);
-
- Call call = calls.get(id);
+ LOG.debug(getName() + " got value #" + callId);
- int state = in.readInt(); // read call status
- if (state == Status.SUCCESS.state) {
+ Call call = calls.get(callId);
+ RpcStatusProto status = response.getStatus();
+ if (status == RpcStatusProto.SUCCESS) {
Writable value = ReflectionUtils.newInstance(valueClass, conf);
value.readFields(in); // read value
call.setRpcResponse(value);
- calls.remove(id);
- } else if (state == Status.ERROR.state) {
+ calls.remove(callId);
+ } else if (status == RpcStatusProto.ERROR) {
call.setException(new RemoteException(WritableUtils.readString(in),
WritableUtils.readString(in)));
- calls.remove(id);
- } else if (state == Status.FATAL.state) {
+ calls.remove(callId);
+ } else if (status == RpcStatusProto.FATAL) {
// Close the connection
markClosed(new RemoteException(WritableUtils.readString(in),
WritableUtils.readString(in)));
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Sat May 12 20:52:34 2012
@@ -217,7 +217,7 @@ public abstract class Server {
public static final Log AUDITLOG =
LogFactory.getLog("SecurityLogger."+Server.class.getName());
private static final String AUTH_FAILED_FOR = "Auth failed for ";
- private static final String AUTH_SUCCESSFULL_FOR = "Auth successfull for ";
+ private static final String AUTH_SUCCESSFUL_FOR = "Auth successful for ";
private static final ThreadLocal<Server> SERVER = new ThreadLocal<Server>();
@@ -1234,7 +1234,7 @@ public abstract class Server {
LOG.debug("SASL server successfully authenticated client: " + user);
}
rpcMetrics.incrAuthenticationSuccesses();
- AUDITLOG.info(AUTH_SUCCESSFULL_FOR + user);
+ AUDITLOG.info(AUTH_SUCCESSFUL_FOR + user);
saslContextEstablished = true;
}
} else {
@@ -1339,7 +1339,7 @@ public abstract class Server {
+ CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION
+ ") is configured as simple. Please configure another method "
+ "like kerberos or digest.");
- setupResponse(authFailedResponse, authFailedCall, Status.FATAL,
+ setupResponse(authFailedResponse, authFailedCall, RpcStatusProto.FATAL,
null, ae.getClass().getName(), ae.getMessage());
responder.doRespond(authFailedCall);
throw ae;
@@ -1420,7 +1420,7 @@ public abstract class Server {
Call fakeCall = new Call(-1, null, this);
// Versions 3 and greater can interpret this exception
// response in the same manner
- setupResponse(buffer, fakeCall, Status.FATAL,
+ setupResponseOldVersionFatal(buffer, fakeCall,
null, VersionMismatch.class.getName(), errMsg);
responder.doRespond(fakeCall);
@@ -1443,7 +1443,7 @@ public abstract class Server {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
Call fakeCall = new Call(-1, null, this);
- setupResponse(buffer, fakeCall, Status.FATAL, null,
+ setupResponse(buffer, fakeCall, RpcStatusProto.FATAL, null,
IpcException.class.getName(), errMsg);
responder.doRespond(fakeCall);
}
@@ -1579,7 +1579,7 @@ public abstract class Server {
new Call(header.getCallId(), null, this);
ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
- setupResponse(responseBuffer, readParamsFailedCall, Status.FATAL, null,
+ setupResponse(responseBuffer, readParamsFailedCall, RpcStatusProto.FATAL, null,
IOException.class.getName(),
"Unknown rpc kind " + header.getRpcKind());
responder.doRespond(readParamsFailedCall);
@@ -1597,7 +1597,7 @@ public abstract class Server {
new Call(header.getCallId(), null, this);
ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
- setupResponse(responseBuffer, readParamsFailedCall, Status.FATAL, null,
+ setupResponse(responseBuffer, readParamsFailedCall, RpcStatusProto.FATAL, null,
t.getClass().getName(),
"IPC server unable to read call parameters: " + t.getMessage());
responder.doRespond(readParamsFailedCall);
@@ -1627,7 +1627,7 @@ public abstract class Server {
rpcMetrics.incrAuthorizationSuccesses();
} catch (AuthorizationException ae) {
rpcMetrics.incrAuthorizationFailures();
- setupResponse(authFailedResponse, authFailedCall, Status.FATAL, null,
+ setupResponse(authFailedResponse, authFailedCall, RpcStatusProto.FATAL, null,
ae.getClass().getName(), ae.getMessage());
responder.doRespond(authFailedCall);
return false;
@@ -1725,8 +1725,8 @@ public abstract class Server {
// responder.doResponse() since setupResponse may use
// SASL to encrypt response data and SASL enforces
// its own message ordering.
- setupResponse(buf, call, (error == null) ? Status.SUCCESS
- : Status.ERROR, value, errorClass, error);
+ setupResponse(buf, call, (error == null) ? RpcStatusProto.SUCCESS
+ : RpcStatusProto.ERROR, value, errorClass, error);
// Discard the large buf and reset it back to smaller size
// to free up heap
@@ -1859,41 +1859,80 @@ public abstract class Server {
/**
* Setup response for the IPC Call.
*
- * @param response buffer to serialize the response into
+ * @param responseBuf buffer to serialize the response into
* @param call {@link Call} to which we are setting up the response
- * @param status {@link Status} of the IPC call
+ * @param status of the IPC call
* @param rv return value for the IPC Call, if the call was successful
* @param errorClass error class, if the the call failed
* @param error error message, if the call failed
* @throws IOException
*/
- private void setupResponse(ByteArrayOutputStream response,
- Call call, Status status,
+ private void setupResponse(ByteArrayOutputStream responseBuf,
+ Call call, RpcStatusProto status,
Writable rv, String errorClass, String error)
throws IOException {
- response.reset();
- DataOutputStream out = new DataOutputStream(response);
- out.writeInt(call.callId); // write call id
- out.writeInt(status.state); // write status
+ responseBuf.reset();
+ DataOutputStream out = new DataOutputStream(responseBuf);
+ RpcResponseHeaderProto.Builder response =
+ RpcResponseHeaderProto.newBuilder();
+ response.setCallId(call.callId);
+ response.setStatus(status);
- if (status == Status.SUCCESS) {
+
+ if (status == RpcStatusProto.SUCCESS) {
try {
+ response.build().writeDelimitedTo(out);
rv.write(out);
} catch (Throwable t) {
LOG.warn("Error serializing call response for call " + call, t);
// Call back to same function - this is OK since the
// buffer is reset at the top, and since status is changed
// to ERROR it won't infinite loop.
- setupResponse(response, call, Status.ERROR,
+ setupResponse(responseBuf, call, RpcStatusProto.ERROR,
null, t.getClass().getName(),
StringUtils.stringifyException(t));
return;
}
} else {
+ if (status == RpcStatusProto.FATAL) {
+ response.setServerIpcVersionNum(Server.CURRENT_VERSION);
+ }
+ response.build().writeDelimitedTo(out);
WritableUtils.writeString(out, errorClass);
WritableUtils.writeString(out, error);
}
if (call.connection.useWrap) {
+ wrapWithSasl(responseBuf, call);
+ }
+ call.setResponse(ByteBuffer.wrap(responseBuf.toByteArray()));
+ }
+
+ /**
+ * Setup response for the IPC Call on Fatal Error from a
+ * client that is using old version of Hadoop.
+ * The response is serialized using the previous protocol's response
+ * layout.
+ *
+ * @param response buffer to serialize the response into
+ * @param call {@link Call} to which we are setting up the response
+ * @param rv return value for the IPC Call, if the call was successful
+ * @param errorClass error class, if the the call failed
+ * @param error error message, if the call failed
+ * @throws IOException
+ */
+ private void setupResponseOldVersionFatal(ByteArrayOutputStream response,
+ Call call,
+ Writable rv, String errorClass, String error)
+ throws IOException {
+ final int OLD_VERSION_FATAL_STATUS = -1;
+ response.reset();
+ DataOutputStream out = new DataOutputStream(response);
+ out.writeInt(call.callId); // write call id
+ out.writeInt(OLD_VERSION_FATAL_STATUS); // write FATAL_STATUS
+ WritableUtils.writeString(out, errorClass);
+ WritableUtils.writeString(out, error);
+
+ if (call.connection.useWrap) {
wrapWithSasl(response, call);
}
call.setResponse(ByteBuffer.wrap(response.toByteArray()));
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java Sat May 12 20:52:34 2012
@@ -148,9 +148,8 @@ public class JMXJsonServlet extends Http
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) {
try {
- // Do the authorization
- if (!HttpServer.hasAdministratorAccess(getServletContext(), request,
- response)) {
+ if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(),
+ request, response)) {
return;
}
JsonGenerator jg = null;
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java Sat May 12 20:52:34 2012
@@ -106,9 +106,8 @@ public class MetricsServlet extends Http
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
- // Do the authorization
- if (!HttpServer.hasAdministratorAccess(getServletContext(), request,
- response)) {
+ if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(),
+ request, response)) {
return;
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java Sat May 12 20:52:34 2012
@@ -140,7 +140,7 @@ public class NetUtils {
/**
* Util method to build socket addr from either:
- * <host>:<post>
+ * <host>:<port>
* <fs>://<host>:<port>/<path>
*/
public static InetSocketAddress createSocketAddr(String target) {
@@ -150,7 +150,7 @@ public class NetUtils {
/**
* Util method to build socket addr from either:
* <host>
- * <host>:<post>
+ * <host>:<port>
* <fs>://<host>:<port>/<path>
*/
public static InetSocketAddress createSocketAddr(String target,
@@ -375,53 +375,44 @@ public class NetUtils {
}
/**
- * Same as getInputStream(socket, socket.getSoTimeout()).<br><br>
+ * Same as <code>getInputStream(socket, socket.getSoTimeout()).</code>
+ * <br><br>
*
- * From documentation for {@link #getInputStream(Socket, long)}:<br>
- * Returns InputStream for the socket. If the socket has an associated
- * SocketChannel then it returns a
- * {@link SocketInputStream} with the given timeout. If the socket does not
- * have a channel, {@link Socket#getInputStream()} is returned. In the later
- * case, the timeout argument is ignored and the timeout set with
- * {@link Socket#setSoTimeout(int)} applies for reads.<br><br>
- *
- * Any socket created using socket factories returned by {@link NetUtils},
- * must use this interface instead of {@link Socket#getInputStream()}.
- *
* @see #getInputStream(Socket, long)
- *
- * @param socket
- * @return InputStream for reading from the socket.
- * @throws IOException
*/
- public static InputStream getInputStream(Socket socket)
+ public static SocketInputWrapper getInputStream(Socket socket)
throws IOException {
return getInputStream(socket, socket.getSoTimeout());
}
-
+
/**
- * Returns InputStream for the socket. If the socket has an associated
- * SocketChannel then it returns a
- * {@link SocketInputStream} with the given timeout. If the socket does not
- * have a channel, {@link Socket#getInputStream()} is returned. In the later
- * case, the timeout argument is ignored and the timeout set with
- * {@link Socket#setSoTimeout(int)} applies for reads.<br><br>
+ * Return a {@link SocketInputWrapper} for the socket and set the given
+ * timeout. If the socket does not have an associated channel, then its socket
+ * timeout will be set to the specified value. Otherwise, a
+ * {@link SocketInputStream} will be created which reads with the configured
+ * timeout.
*
- * Any socket created using socket factories returned by {@link NetUtils},
+ * Any socket created using socket factories returned by {@link #NetUtils},
* must use this interface instead of {@link Socket#getInputStream()}.
- *
+ *
+ * In general, this should be called only once on each socket: see the note
+ * in {@link SocketInputWrapper#setTimeout(long)} for more information.
+ *
* @see Socket#getChannel()
*
* @param socket
- * @param timeout timeout in milliseconds. This may not always apply. zero
- * for waiting as long as necessary.
- * @return InputStream for reading from the socket.
+ * @param timeout timeout in milliseconds. zero for waiting as
+ * long as necessary.
+ * @return SocketInputWrapper for reading from the socket.
* @throws IOException
*/
- public static InputStream getInputStream(Socket socket, long timeout)
+ public static SocketInputWrapper getInputStream(Socket socket, long timeout)
throws IOException {
- return (socket.getChannel() == null) ?
- socket.getInputStream() : new SocketInputStream(socket, timeout);
+ InputStream stm = (socket.getChannel() == null) ?
+ socket.getInputStream() : new SocketInputStream(socket);
+ SocketInputWrapper w = new SocketInputWrapper(socket, stm);
+ w.setTimeout(timeout);
+ return w;
}
/**
@@ -503,7 +494,7 @@ public class NetUtils {
* also takes a local address and port to bind the socket to.
*
* @param socket
- * @param address the remote address
+ * @param endpoint the remote address
* @param localAddr the local address to bind the socket to
* @param timeout timeout in milliseconds
*/
@@ -558,16 +549,11 @@ public class NetUtils {
* @return its IP address in the string format
*/
public static String normalizeHostName(String name) {
- if (Character.digit(name.charAt(0), 10) != -1) { // it is an IP
+ try {
+ return InetAddress.getByName(name).getHostAddress();
+ } catch (UnknownHostException e) {
return name;
- } else {
- try {
- InetAddress ipAddress = InetAddress.getByName(name);
- return ipAddress.getHostAddress();
- } catch (UnknownHostException e) {
- return name;
- }
- }
+ }
}
/**
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java Sat May 12 20:52:34 2012
@@ -247,6 +247,10 @@ abstract class SocketIOWithTimeout {
ops));
}
}
+
+ public void setTimeout(long timeoutMs) {
+ this.timeout = timeoutMs;
+ }
private static String timeoutExceptionString(SelectableChannel channel,
long timeout, int ops) {
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java Sat May 12 20:52:34 2012
@@ -28,9 +28,6 @@ import java.nio.channels.ReadableByteCha
import java.nio.channels.SelectableChannel;
import java.nio.channels.SelectionKey;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
/**
* This implements an input stream that can have a timeout while reading.
* This sets non-blocking flag on the socket channel.
@@ -40,9 +37,7 @@ import org.apache.hadoop.classification.
* IllegalBlockingModeException.
* Please use {@link SocketOutputStream} for writing.
*/
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
-@InterfaceStability.Unstable
-public class SocketInputStream extends InputStream
+class SocketInputStream extends InputStream
implements ReadableByteChannel {
private Reader reader;
@@ -171,4 +166,8 @@ public class SocketInputStream extends I
public void waitForReadable() throws IOException {
reader.waitForIO(SelectionKey.OP_READ);
}
+
+ public void setTimeout(long timeoutMs) {
+ reader.setTimeout(timeoutMs);
+ }
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java Sat May 12 20:52:34 2012
@@ -17,14 +17,11 @@
package org.apache.hadoop.security;
import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
+import java.net.URLConnection;
import java.net.UnknownHostException;
import java.security.AccessController;
import java.security.PrivilegedAction;
@@ -45,6 +42,8 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenInfo;
@@ -135,79 +134,6 @@ public class SecurityUtil {
}
/**
- * Explicitly pull the service ticket for the specified host. This solves a
- * problem with Java's Kerberos SSL problem where the client cannot
- * authenticate against a cross-realm service. It is necessary for clients
- * making kerberized https requests to call this method on the target URL
- * to ensure that in a cross-realm environment the remote host will be
- * successfully authenticated.
- *
- * This method is internal to Hadoop and should not be used by other
- * applications. This method should not be considered stable or open:
- * it will be removed when the Java behavior is changed.
- *
- * @param remoteHost Target URL the krb-https client will access
- * @throws IOException if the service ticket cannot be retrieved
- */
- public static void fetchServiceTicket(URL remoteHost) throws IOException {
- if(!UserGroupInformation.isSecurityEnabled())
- return;
-
- String serviceName = "host/" + remoteHost.getHost();
- if (LOG.isDebugEnabled())
- LOG.debug("Fetching service ticket for host at: " + serviceName);
- Object serviceCred = null;
- Method credsToTicketMeth;
- Class<?> krb5utilClass;
- try {
- Class<?> principalClass;
- Class<?> credentialsClass;
-
- if (System.getProperty("java.vendor").contains("IBM")) {
- principalClass = Class.forName("com.ibm.security.krb5.PrincipalName");
-
- credentialsClass = Class.forName("com.ibm.security.krb5.Credentials");
- krb5utilClass = Class.forName("com.ibm.security.jgss.mech.krb5");
- } else {
- principalClass = Class.forName("sun.security.krb5.PrincipalName");
- credentialsClass = Class.forName("sun.security.krb5.Credentials");
- krb5utilClass = Class.forName("sun.security.jgss.krb5.Krb5Util");
- }
- @SuppressWarnings("rawtypes")
- Constructor principalConstructor = principalClass.getConstructor(String.class,
- int.class);
- Field KRB_NT_SRV_HST = principalClass.getDeclaredField("KRB_NT_SRV_HST");
- Method acquireServiceCredsMeth =
- credentialsClass.getDeclaredMethod("acquireServiceCreds",
- String.class, credentialsClass);
- Method ticketToCredsMeth = krb5utilClass.getDeclaredMethod("ticketToCreds",
- KerberosTicket.class);
- credsToTicketMeth = krb5utilClass.getDeclaredMethod("credsToTicket",
- credentialsClass);
-
- Object principal = principalConstructor.newInstance(serviceName,
- KRB_NT_SRV_HST.get(principalClass));
-
- serviceCred = acquireServiceCredsMeth.invoke(credentialsClass,
- principal.toString(),
- ticketToCredsMeth.invoke(krb5utilClass, getTgtFromSubject()));
- } catch (Exception e) {
- throw new IOException("Can't get service ticket for: "
- + serviceName, e);
- }
- if (serviceCred == null) {
- throw new IOException("Can't get service ticket for " + serviceName);
- }
- try {
- Subject.getSubject(AccessController.getContext()).getPrivateCredentials()
- .add(credsToTicketMeth.invoke(krb5utilClass, serviceCred));
- } catch (Exception e) {
- throw new IOException("Can't get service ticket for: "
- + serviceName, e);
- }
- }
-
- /**
* Convert Kerberos principal name pattern to valid Kerberos principal
* names. It replaces hostname pattern with hostname, which should be
* fully-qualified domain name. If hostname is null or "0.0.0.0", it uses
@@ -514,6 +440,30 @@ public class SecurityUtil {
}
/**
+ * Open a (if need be) secure connection to a URL in a secure environment
+ * that is using SPNEGO to authenticate its URLs. All Namenode and Secondary
+ * Namenode URLs that are protected via SPNEGO should be accessed via this
+ * method.
+ *
+ * @param url to authenticate via SPNEGO.
+ * @return A connection that has been authenticated via SPNEGO
+ * @throws IOException If unable to authenticate via SPNEGO
+ */
+ public static URLConnection openSecureHttpConnection(URL url) throws IOException {
+ if(!UserGroupInformation.isSecurityEnabled()) {
+ return url.openConnection();
+ }
+
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ try {
+ return new AuthenticatedURL().openConnection(url, token);
+ } catch (AuthenticationException e) {
+ throw new IOException("Exception trying to open authenticated connection to "
+ + url, e);
+ }
+ }
+
+ /**
* Resolves a host subject to the security requirements determined by
* hadoop.security.token.service.use_ip.
*
@@ -664,10 +614,4 @@ public class SecurityUtil {
}
}
- public static void initKrb5CipherSuites() {
- if (UserGroupInformation.isSecurityEnabled()) {
- System.setProperty("https.cipherSuites",
- Krb5AndCertsSslSocketConnector.KRB5_CIPHER_SUITES.get(0));
- }
- }
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java?rev=1337645&r1=1337644&r2=1337645&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java Sat May 12 20:52:34 2012
@@ -59,7 +59,7 @@ public class ServiceAuthorizationManager
public static final Log AUDITLOG =
LogFactory.getLog("SecurityLogger."+ServiceAuthorizationManager.class.getName());
- private static final String AUTHZ_SUCCESSFULL_FOR = "Authorization successfull for ";
+ private static final String AUTHZ_SUCCESSFUL_FOR = "Authorization successful for ";
private static final String AUTHZ_FAILED_FOR = "Authorization failed for ";
@@ -108,7 +108,7 @@ public class ServiceAuthorizationManager
" is not authorized for protocol " + protocol +
", expected client Kerberos principal is " + clientPrincipal);
}
- AUDITLOG.info(AUTHZ_SUCCESSFULL_FOR + user + " for protocol="+protocol);
+ AUDITLOG.info(AUTHZ_SUCCESSFUL_FOR + user + " for protocol="+protocol);
}
public synchronized void refresh(Configuration conf,