You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2012/08/10 00:30:32 UTC
svn commit: r1371518 - in
/hadoop/common/branches/HDFS-3077/hadoop-common-project:
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/
hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/
hadoop-common/ ha...
Author: todd
Date: Thu Aug 9 22:29:36 2012
New Revision: 1371518
URL: http://svn.apache.org/viewvc?rev=1371518&view=rev
Log:
Merge trunk into branch.
Branch will not build after this commit: need to implement new JournalManager
interfaces in QuorumJournalManager in a follow-up.
Added:
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/ConnectionConfigurator.java
- copied unchanged from r1371513, hadoop/common/trunk/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/ConnectionConfigurator.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/JNIFlags.cmake
- copied unchanged from r1371513, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/JNIFlags.cmake
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java
- copied unchanged from r1371513, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java
Modified:
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/Authenticator.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/CMakeLists.txt
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslInputStream.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java Thu Aug 9 22:29:36 2012
@@ -171,6 +171,7 @@ public class AuthenticatedURL {
}
private Authenticator authenticator;
+ private ConnectionConfigurator connConfigurator;
/**
* Creates an {@link AuthenticatedURL}.
@@ -186,11 +187,25 @@ public class AuthenticatedURL {
* KerberosAuthenticator} is used.
*/
public AuthenticatedURL(Authenticator authenticator) {
+ this(authenticator, null);
+ }
+
+ /**
+ * Creates an <code>AuthenticatedURL</code>.
+ *
+ * @param authenticator the {@link Authenticator} instance to use, if <code>null</code> a {@link
+ * KerberosAuthenticator} is used.
+ * @param connConfigurator a connection configurator.
+ */
+ public AuthenticatedURL(Authenticator authenticator,
+ ConnectionConfigurator connConfigurator) {
try {
this.authenticator = (authenticator != null) ? authenticator : DEFAULT_AUTHENTICATOR.newInstance();
} catch (Exception ex) {
throw new RuntimeException(ex);
}
+ this.connConfigurator = connConfigurator;
+ this.authenticator.setConnectionConfigurator(connConfigurator);
}
/**
@@ -216,6 +231,9 @@ public class AuthenticatedURL {
}
authenticator.authenticate(url, token);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ if (connConfigurator != null) {
+ conn = connConfigurator.configure(conn);
+ }
injectToken(conn, token);
return conn;
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/Authenticator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/Authenticator.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/Authenticator.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/Authenticator.java Thu Aug 9 22:29:36 2012
@@ -25,6 +25,14 @@ import java.net.URL;
public interface Authenticator {
/**
+ * Sets a {@link ConnectionConfigurator} instance to use for
+ * configuring connections.
+ *
+ * @param configurator the {@link ConnectionConfigurator} instance.
+ */
+ public void setConnectionConfigurator(ConnectionConfigurator configurator);
+
+ /**
* Authenticates against a URL and returns a {@link AuthenticatedURL.Token} to be
* used by subsequent requests.
*
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java Thu Aug 9 22:29:36 2012
@@ -113,6 +113,18 @@ public class KerberosAuthenticator imple
private URL url;
private HttpURLConnection conn;
private Base64 base64;
+ private ConnectionConfigurator connConfigurator;
+
+ /**
+ * Sets a {@link ConnectionConfigurator} instance to use for
+ * configuring connections.
+ *
+ * @param configurator the {@link ConnectionConfigurator} instance.
+ */
+ @Override
+ public void setConnectionConfigurator(ConnectionConfigurator configurator) {
+ connConfigurator = configurator;
+ }
/**
* Performs SPNEGO authentication against the specified URL.
@@ -135,6 +147,9 @@ public class KerberosAuthenticator imple
this.url = url;
base64 = new Base64(0);
conn = (HttpURLConnection) url.openConnection();
+ if (connConfigurator != null) {
+ conn = connConfigurator.configure(conn);
+ }
conn.setRequestMethod(AUTH_HTTP_METHOD);
conn.connect();
if (isNegotiate()) {
@@ -244,6 +259,9 @@ public class KerberosAuthenticator imple
private void sendToken(byte[] outToken) throws IOException, AuthenticationException {
String token = base64.encodeToString(outToken);
conn = (HttpURLConnection) url.openConnection();
+ if (connConfigurator != null) {
+ conn = connConfigurator.configure(conn);
+ }
conn.setRequestMethod(AUTH_HTTP_METHOD);
conn.setRequestProperty(AUTHORIZATION, NEGOTIATE + " " + token);
conn.connect();
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java Thu Aug 9 22:29:36 2012
@@ -32,6 +32,19 @@ public class PseudoAuthenticator impleme
private static final String USER_NAME_EQ = USER_NAME + "=";
+ private ConnectionConfigurator connConfigurator;
+
+ /**
+ * Sets a {@link ConnectionConfigurator} instance to use for
+ * configuring connections.
+ *
+ * @param configurator the {@link ConnectionConfigurator} instance.
+ */
+ @Override
+ public void setConnectionConfigurator(ConnectionConfigurator configurator) {
+ connConfigurator = configurator;
+ }
+
/**
* Performs simple authentication against the specified URL.
* <p/>
@@ -56,6 +69,9 @@ public class PseudoAuthenticator impleme
strUrl += paramSeparator + USER_NAME_EQ + getUserName();
url = new URL(strUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ if (connConfigurator != null) {
+ conn = connConfigurator.configure(conn);
+ }
conn.setRequestMethod("OPTIONS");
conn.connect();
AuthenticatedURL.extractToken(conn, token);
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java Thu Aug 9 22:29:36 2012
@@ -13,8 +13,10 @@
*/
package org.apache.hadoop.security.authentication.client;
+import junit.framework.Assert;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import junit.framework.TestCase;
+import org.mockito.Mockito;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.servlet.Context;
import org.mortbay.jetty.servlet.FilterHolder;
@@ -113,6 +115,18 @@ public abstract class AuthenticatorTestC
return "http://" + host + ":" + port + "/foo/bar";
}
+ private static class TestConnectionConfigurator
+ implements ConnectionConfigurator {
+ boolean invoked;
+
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ invoked = true;
+ return conn;
+ }
+ }
+
private String POST = "test";
protected void _testAuthentication(Authenticator authenticator, boolean doPost) throws Exception {
@@ -120,8 +134,10 @@ public abstract class AuthenticatorTestC
try {
URL url = new URL(getBaseURL());
AuthenticatedURL.Token token = new AuthenticatedURL.Token();
- AuthenticatedURL aUrl = new AuthenticatedURL(authenticator);
+ TestConnectionConfigurator connConf = new TestConnectionConfigurator();
+ AuthenticatedURL aUrl = new AuthenticatedURL(authenticator, connConf);
HttpURLConnection conn = aUrl.openConnection(url, token);
+ Assert.assertTrue(connConf.invoked);
String tokenStr = token.toString();
if (doPost) {
conn.setRequestMethod("POST");
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java Thu Aug 9 22:29:36 2012
@@ -18,6 +18,7 @@ import junit.framework.TestCase;
import org.mockito.Mockito;
import java.net.HttpURLConnection;
+import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -114,4 +115,21 @@ public class TestAuthenticatedURL extend
}
}
+ public void testConnectionConfigurator() throws Exception {
+ HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
+ Mockito.when(conn.getResponseCode()).
+ thenReturn(HttpURLConnection.HTTP_UNAUTHORIZED);
+
+ ConnectionConfigurator connConf =
+ Mockito.mock(ConnectionConfigurator.class);
+ Mockito.when(connConf.configure(Mockito.<HttpURLConnection>any())).
+ thenReturn(conn);
+
+ Authenticator authenticator = Mockito.mock(Authenticator.class);
+
+ AuthenticatedURL aURL = new AuthenticatedURL(authenticator, connConf);
+ aURL.openConnection(new URL("http://foo"), new AuthenticatedURL.Token());
+ Mockito.verify(connConf).configure(Mockito.<HttpURLConnection>any());
+ }
+
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 9 22:29:36 2012
@@ -211,6 +211,8 @@ Branch-2 ( Unreleased changes )
HADOOP-8465. hadoop-auth should support ephemeral authentication (tucu)
+ HADOOP-8644. AuthenticatedURL should be able to use SSLFactory. (tucu)
+
IMPROVEMENTS
HADOOP-8340. SNAPSHOT build versions should compare as less than their eventual
@@ -277,6 +279,9 @@ Branch-2 ( Unreleased changes )
HADOOP-8609. IPC server logs a useless message when shutting down socket.
(Jon Zuanich via atm)
+ HADOOP-8620. Add -Drequire.fuse and -Drequire.snappy. (Colin
+ Patrick McCabe via eli)
+
BUG FIXES
HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname
@@ -371,6 +376,12 @@ Branch-2 ( Unreleased changes )
hadoop.security.group.mapping.ldap.search.filter.user. (Jonathan Natkins
via atm)
+ HADOOP-8480. The native build should honor -DskipTests.
+ (Colin Patrick McCabe via eli)
+
+ HADOOP-8659. Native libraries must build with soft-float ABI for Oracle JVM
+ on ARM. (Trevor Robinson via todd)
+
BREAKDOWN OF HDFS-3042 SUBTASKS
HADOOP-8220. ZKFailoverController doesn't handle failure to become active
@@ -403,6 +414,8 @@ Branch-2 ( Unreleased changes )
HADOOP-8405. ZKFC tests leak ZK instances. (todd)
+ HADOOP-8660. TestPseudoAuthenticator failing with NPE. (tucu)
+
Release 2.0.0-alpha - 05-23-2012
INCOMPATIBLE CHANGES
@@ -867,6 +880,15 @@ Release 0.23.3 - UNRELEASED
HADOOP-8635. Cannot cancel paths registered deleteOnExit (daryn via bobby)
+ HADOOP-8637. FilterFileSystem#setWriteChecksum is broken (daryn via bobby)
+
+ HADOOP-8370. Native build failure: javah: class file for
+ org.apache.hadoop.classification.InterfaceAudience not found (Trevor
+ Robinson via tgraves)
+
+ HADOOP-8633. Interrupted FsShell copies may leave tmp files (Daryn Sharp
+ via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1367365-1371513
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/pom.xml?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/pom.xml Thu Aug 9 22:29:36 2012
@@ -31,9 +31,6 @@
<packaging>jar</packaging>
<properties>
- <snappy.prefix>/usr/local</snappy.prefix>
- <snappy.lib>${snappy.prefix}/lib</snappy.lib>
- <bundle.snappy>false</bundle.snappy>
<kdc.resource.dir>src/test/resources/kdc</kdc.resource.dir>
<hadoop.component>common</hadoop.component>
<is.hadoop.component>true</is.hadoop.component>
@@ -44,7 +41,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
- <scope>provided</scope>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -533,10 +530,10 @@
<activeByDefault>false</activeByDefault>
</activation>
<properties>
- <snappy.prefix>/usr/local</snappy.prefix>
- <snappy.lib>${snappy.prefix}/lib</snappy.lib>
- <snappy.include>${snappy.prefix}/include</snappy.include>
- <runas.home></runas.home>
+ <snappy.prefix></snappy.prefix>
+ <snappy.lib></snappy.lib>
+ <snappy.include></snappy.include>
+ <require.snappy>false</require.snappy>
</properties>
<build>
<plugins>
@@ -579,9 +576,7 @@
<configuration>
<target>
<exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
- <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
- <env key="CFLAGS" value="-I${snappy.include}"/>
- <env key="LDFLAGS" value="-L${snappy.lib}"/>
+ <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/>
</exec>
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
<arg line="VERBOSE=1"/>
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/CMakeLists.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/CMakeLists.txt?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/CMakeLists.txt (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/CMakeLists.txt Thu Aug 9 22:29:36 2012
@@ -21,18 +21,7 @@ cmake_minimum_required(VERSION 2.6 FATAL
# Default to release builds
set(CMAKE_BUILD_TYPE, Release)
-# If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit.
-# This variable is set by maven.
-if (JVM_ARCH_DATA_MODEL EQUAL 32)
- # force 32-bit code generation on amd64/x86_64, ppc64, sparc64
- if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_SYSTEM_PROCESSOR MATCHES ".*64")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
- set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32")
- endif ()
- if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
- set(CMAKE_SYSTEM_PROCESSOR "i686")
- endif ()
-endif (JVM_ARCH_DATA_MODEL EQUAL 32)
+include(JNIFlags.cmake NO_POLICY_SCOPE)
# Compile a library with both shared and static variants
function(add_dual_library LIBNAME)
@@ -79,17 +68,26 @@ INCLUDE(CheckCSourceCompiles)
CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE)
CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE)
-find_library(SNAPPY_LIBRARY NAMES snappy PATHS)
-find_path(SNAPPY_INCLUDE_DIR NAMES snappy.h PATHS)
-if (SNAPPY_LIBRARY)
+find_library(SNAPPY_LIBRARY
+ NAMES snappy
+ PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/lib
+ ${CUSTOM_SNAPPY_PREFIX}/lib64 ${CUSTOM_SNAPPY_LIB})
+find_path(SNAPPY_INCLUDE_DIR
+ NAMES snappy.h
+ PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/include
+ ${CUSTOM_SNAPPY_INCLUDE})
+if (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
GET_FILENAME_COMPONENT(HADOOP_SNAPPY_LIBRARY ${SNAPPY_LIBRARY} NAME)
set(SNAPPY_SOURCE_FILES
"${D}/io/compress/snappy/SnappyCompressor.c"
"${D}/io/compress/snappy/SnappyDecompressor.c")
-else (${SNAPPY_LIBRARY})
+else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
set(SNAPPY_INCLUDE_DIR "")
set(SNAPPY_SOURCE_FILES "")
-endif (SNAPPY_LIBRARY)
+ IF(REQUIRE_SNAPPY)
+ MESSAGE(FATAL_ERROR "Required snappy library could not be found. SNAPPY_LIBRARY=${SNAPPY_LIBRARY}, SNAPPY_INCLUDE_DIR=${SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_INCLUDE_DIR=${CUSTOM_SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_PREFIX=${CUSTOM_SNAPPY_PREFIX}, CUSTOM_SNAPPY_INCLUDE=${CUSTOM_SNAPPY_INCLUDE}")
+ ENDIF(REQUIRE_SNAPPY)
+endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
include_directories(
${GENERATED_JAVAH}
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh Thu Aug 9 22:29:36 2012
@@ -42,8 +42,8 @@ HADOOP_COMMON_LIB_JARS_DIR=${HADOOP_COMM
HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_COMMON_LIB_NATIVE_DIR:-"lib/native"}
HDFS_DIR=${HDFS_DIR:-"share/hadoop/hdfs"}
HDFS_LIB_JARS_DIR=${HDFS_LIB_JARS_DIR:-"share/hadoop/hdfs/lib"}
-YARN_DIR=${YARN_DIR:-"share/hadoop/mapreduce"}
-YARN_LIB_JARS_DIR=${YARN_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"}
+YARN_DIR=${YARN_DIR:-"share/hadoop/yarn"}
+YARN_LIB_JARS_DIR=${YARN_LIB_JARS_DIR:-"share/hadoop/yarn/lib"}
MAPRED_DIR=${MAPRED_DIR:-"share/hadoop/mapreduce"}
MAPRED_LIB_JARS_DIR=${MAPRED_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"}
Propchange: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1367365-1371513
Propchange: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1367365-1371513
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Thu Aug 9 22:29:36 2012
@@ -620,7 +620,8 @@ public abstract class FileSystem extends
conf.getInt("io.bytes.per.checksum", 512),
64 * 1024,
getDefaultReplication(),
- conf.getInt("io.file.buffer.size", 4096));
+ conf.getInt("io.file.buffer.size", 4096),
+ false);
}
/**
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java Thu Aug 9 22:29:36 2012
@@ -376,7 +376,7 @@ public class FilterFileSystem extends Fi
@Override
public void setWriteChecksum(boolean writeChecksum) {
- fs.setVerifyChecksum(writeChecksum);
+ fs.setWriteChecksum(writeChecksum);
}
@Override
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java Thu Aug 9 22:29:36 2012
@@ -48,17 +48,20 @@ public class FsServerDefaults implements
private int writePacketSize;
private short replication;
private int fileBufferSize;
+ private boolean encryptDataTransfer;
public FsServerDefaults() {
}
public FsServerDefaults(long blockSize, int bytesPerChecksum,
- int writePacketSize, short replication, int fileBufferSize) {
+ int writePacketSize, short replication, int fileBufferSize,
+ boolean encryptDataTransfer) {
this.blockSize = blockSize;
this.bytesPerChecksum = bytesPerChecksum;
this.writePacketSize = writePacketSize;
this.replication = replication;
this.fileBufferSize = fileBufferSize;
+ this.encryptDataTransfer = encryptDataTransfer;
}
public long getBlockSize() {
@@ -80,6 +83,10 @@ public class FsServerDefaults implements
public int getFileBufferSize() {
return fileBufferSize;
}
+
+ public boolean getEncryptDataTransfer() {
+ return encryptDataTransfer;
+ }
// /////////////////////////////////////////
// Writable
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java Thu Aug 9 22:29:36 2012
@@ -44,6 +44,7 @@ public class FtpConfigKeys extends Commo
public static final String CLIENT_WRITE_PACKET_SIZE_KEY =
"ftp.client-write-packet-size";
public static final int CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024;
+ public static final boolean ENCRYPT_DATA_TRANSFER_DEFAULT = false;
protected static FsServerDefaults getServerDefaults() throws IOException {
return new FsServerDefaults(
@@ -51,7 +52,8 @@ public class FtpConfigKeys extends Commo
BYTES_PER_CHECKSUM_DEFAULT,
CLIENT_WRITE_PACKET_SIZE_DEFAULT,
REPLICATION_DEFAULT,
- STREAM_BUFFER_SIZE_DEFAULT);
+ STREAM_BUFFER_SIZE_DEFAULT,
+ ENCRYPT_DATA_TRANSFER_DEFAULT);
}
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java Thu Aug 9 22:29:36 2012
@@ -43,6 +43,7 @@ public class LocalConfigKeys extends Com
public static final String CLIENT_WRITE_PACKET_SIZE_KEY =
"file.client-write-packet-size";
public static final int CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024;
+ public static final boolean ENCRYPT_DATA_TRANSFER_DEFAULT = false;
public static FsServerDefaults getServerDefaults() throws IOException {
return new FsServerDefaults(
@@ -50,7 +51,8 @@ public class LocalConfigKeys extends Com
BYTES_PER_CHECKSUM_DEFAULT,
CLIENT_WRITE_PACKET_SIZE_DEFAULT,
REPLICATION_DEFAULT,
- STREAM_BUFFER_SIZE_DEFAULT);
+ STREAM_BUFFER_SIZE_DEFAULT,
+ ENCRYPT_DATA_TRANSFER_DEFAULT);
}
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Thu Aug 9 22:29:36 2012
@@ -24,6 +24,8 @@ import java.io.InputStream;
import java.util.LinkedList;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException;
import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
@@ -232,31 +234,65 @@ abstract class CommandWithDestination ex
if (target.exists && (target.stat.isDirectory() || !overwrite)) {
throw new PathExistsException(target.toString());
}
- target.fs.setWriteChecksum(writeChecksum);
- PathData tempFile = null;
+ TargetFileSystem targetFs = new TargetFileSystem(target.fs);
try {
- tempFile = target.createTempFile(target+"._COPYING_");
- FSDataOutputStream out = target.fs.create(tempFile.path, true);
- IOUtils.copyBytes(in, out, getConf(), true);
+ PathData tempTarget = target.suffix("._COPYING_");
+ targetFs.setWriteChecksum(writeChecksum);
+ targetFs.writeStreamToFile(in, tempTarget);
+ targetFs.rename(tempTarget, target);
+ } finally {
+ targetFs.close(); // last ditch effort to ensure temp file is removed
+ }
+ }
+
+ // Helper filter filesystem that registers created files as temp files to
+ // be deleted on exit unless successfully renamed
+ private static class TargetFileSystem extends FilterFileSystem {
+ TargetFileSystem(FileSystem fs) {
+ super(fs);
+ }
+
+ void writeStreamToFile(InputStream in, PathData target) throws IOException {
+ FSDataOutputStream out = null;
+ try {
+ out = create(target);
+ IOUtils.copyBytes(in, out, getConf(), true);
+ } finally {
+ IOUtils.closeStream(out); // just in case copyBytes didn't
+ }
+ }
+
+ // tag created files as temp files
+ FSDataOutputStream create(PathData item) throws IOException {
+ try {
+ return create(item.path, true);
+ } finally { // might have been created but stream was interrupted
+ deleteOnExit(item.path);
+ }
+ }
+
+ void rename(PathData src, PathData target) throws IOException {
// the rename method with an option to delete the target is deprecated
- if (target.exists && !target.fs.delete(target.path, false)) {
+ if (target.exists && !delete(target.path, false)) {
// too bad we don't know why it failed
PathIOException e = new PathIOException(target.toString());
e.setOperation("delete");
throw e;
}
- if (!tempFile.fs.rename(tempFile.path, target.path)) {
+ if (!rename(src.path, target.path)) {
// too bad we don't know why it failed
- PathIOException e = new PathIOException(tempFile.toString());
+ PathIOException e = new PathIOException(src.toString());
e.setOperation("rename");
e.setTargetPath(target.toString());
throw e;
}
- tempFile = null;
- } finally {
- if (tempFile != null) {
- tempFile.fs.delete(tempFile.path, false);
- }
+ // cancel delete on exit if rename is successful
+ cancelDeleteOnExit(src.path);
+ }
+ @Override
+ public void close() {
+ // purge any remaining temp files, but don't close underlying fs
+ processDeleteOnExit();
}
}
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java Thu Aug 9 22:29:36 2012
@@ -60,7 +60,7 @@ public class PathData implements Compara
* @throws IOException if anything goes wrong...
*/
public PathData(String pathString, Configuration conf) throws IOException {
- this(FileSystem.get(URI.create(pathString), conf), pathString);
+ this(FileSystem.get(stringToUri(pathString), conf), pathString);
}
/**
@@ -170,16 +170,13 @@ public class PathData implements Compara
}
/**
- * Returns a temporary file for this PathData with the given extension.
- * The file will be deleted on exit.
- * @param extension for the temporary file
+ * Returns a new PathData with the given extension.
+ * @param extension for the suffix
* @return PathData
* @throws IOException shouldn't happen
*/
- public PathData createTempFile(String extension) throws IOException {
- PathData tmpFile = new PathData(fs, uri+"._COPYING_");
- fs.deleteOnExit(tmpFile.path);
- return tmpFile;
+ public PathData suffix(String extension) throws IOException {
+ return new PathData(fs, this+extension);
}
/**
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslInputStream.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslInputStream.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslInputStream.java Thu Aug 9 22:29:36 2012
@@ -22,6 +22,8 @@ import java.io.DataInputStream;
import java.io.EOFException;
import java.io.InputStream;
import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
import javax.security.sasl.Sasl;
import javax.security.sasl.SaslClient;
@@ -42,7 +44,7 @@ import org.apache.hadoop.classification.
*/
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Evolving
-public class SaslInputStream extends InputStream {
+public class SaslInputStream extends InputStream implements ReadableByteChannel {
public static final Log LOG = LogFactory.getLog(SaslInputStream.class);
private final DataInputStream inStream;
@@ -65,6 +67,8 @@ public class SaslInputStream extends Inp
private int ostart = 0;
// position of the last "new" byte
private int ofinish = 0;
+ // whether or not this stream is open
+ private boolean isOpen = true;
private static int unsignedBytesToInt(byte[] buf) {
if (buf.length != 4) {
@@ -330,6 +334,7 @@ public class SaslInputStream extends Inp
ostart = 0;
ofinish = 0;
inStream.close();
+ isOpen = false;
}
/**
@@ -342,4 +347,28 @@ public class SaslInputStream extends Inp
public boolean markSupported() {
return false;
}
+
+ @Override
+ public boolean isOpen() {
+ return isOpen;
+ }
+
+ @Override
+ public int read(ByteBuffer dst) throws IOException {
+ int bytesRead = 0;
+ if (dst.hasArray()) {
+ bytesRead = read(dst.array(), dst.arrayOffset() + dst.position(),
+ dst.remaining());
+ if (bytesRead > -1) {
+ dst.position(dst.position() + bytesRead);
+ }
+ } else {
+ byte[] buf = new byte[dst.remaining()];
+ bytesRead = read(buf);
+ if (bytesRead > -1) {
+ dst.put(buf, 0, bytesRead);
+ }
+ }
+ return bytesRead;
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java Thu Aug 9 22:29:36 2012
@@ -20,14 +20,17 @@ package org.apache.hadoop.security.ssl;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.util.ReflectionUtils;
import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLServerSocketFactory;
import javax.net.ssl.SSLSocketFactory;
import java.io.IOException;
+import java.net.HttpURLConnection;
import java.security.GeneralSecurityException;
/**
@@ -42,7 +45,7 @@ import java.security.GeneralSecurityExce
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public class SSLFactory {
+public class SSLFactory implements ConnectionConfigurator {
@InterfaceAudience.Private
public static enum Mode { CLIENT, SERVER }
@@ -234,4 +237,29 @@ public class SSLFactory {
return requireClientCert;
}
+ /**
+ * If the given {@link HttpURLConnection} is an {@link HttpsURLConnection}
+ * configures the connection with the {@link SSLSocketFactory} and
+ * {@link HostnameVerifier} of this SSLFactory, otherwise does nothing.
+ *
+ * @param conn the {@link HttpURLConnection} instance to configure.
+ * @return the configured {@link HttpURLConnection} instance.
+ *
+ * @throws IOException if an IO error occurred.
+ */
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ if (conn instanceof HttpsURLConnection) {
+ HttpsURLConnection sslConn = (HttpsURLConnection) conn;
+ try {
+ sslConn.setSSLSocketFactory(createSSLSocketFactory());
+ } catch (GeneralSecurityException ex) {
+ throw new IOException(ex);
+ }
+ sslConn.setHostnameVerifier(getHostnameVerifier());
+ conn = sslConn;
+ }
+ return conn;
+ }
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Thu Aug 9 22:29:36 2012
@@ -196,7 +196,9 @@ Deprecated Properties
*---+---+
|mapred.compress.map.output | mapreduce.map.output.compress
*---+---+
-|mapred.create.symlink | mapreduce.job.cache.symlink.create
+|mapred.create.symlink | NONE - symlinking is always on
+*---+---+
+|mapreduce.job.cache.symlink.create | NONE - symlinking is always on
*---+---+
|mapred.data.field.separator | mapreduce.fieldsel.data.field.separator
*---+---+
Propchange: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1367365-1371513
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java Thu Aug 9 22:29:36 2012
@@ -286,6 +286,30 @@ public class TestFilterFileSystem {
checkFsConf(flfs, conf, 3);
}
+ @Test
+ public void testVerifyChecksumPassthru() {
+ FileSystem mockFs = mock(FileSystem.class);
+ FileSystem fs = new FilterFileSystem(mockFs);
+
+ fs.setVerifyChecksum(false);
+ verify(mockFs).setVerifyChecksum(eq(false));
+ reset(mockFs);
+ fs.setVerifyChecksum(true);
+ verify(mockFs).setVerifyChecksum(eq(true));
+ }
+
+ @Test
+ public void testWriteChecksumPassthru() {
+ FileSystem mockFs = mock(FileSystem.class);
+ FileSystem fs = new FilterFileSystem(mockFs);
+
+ fs.setWriteChecksum(false);
+ verify(mockFs).setWriteChecksum(eq(false));
+ reset(mockFs);
+ fs.setWriteChecksum(true);
+ verify(mockFs).setWriteChecksum(eq(true));
+ }
+
private void checkInit(FilterFileSystem fs, boolean expectInit)
throws Exception {
URI uri = URI.create("filter:/");
Modified: hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java?rev=1371518&r1=1371517&r2=1371518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java Thu Aug 9 22:29:36 2012
@@ -25,6 +25,7 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import javax.net.ssl.HttpsURLConnection;
import java.io.File;
import java.net.URL;
import java.security.GeneralSecurityException;
@@ -161,4 +162,23 @@ public class TestSSLFactory {
}
}
+ @Test
+ public void testConnectionConfigurator() throws Exception {
+ Configuration conf = createConfiguration(false);
+ conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "STRICT_IE6");
+ SSLFactory sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+ try {
+ sslFactory.init();
+ HttpsURLConnection sslConn =
+ (HttpsURLConnection) new URL("https://foo").openConnection();
+ Assert.assertNotSame("STRICT_IE6",
+ sslConn.getHostnameVerifier().toString());
+ sslFactory.configure(sslConn);
+ Assert.assertEquals("STRICT_IE6",
+ sslConn.getHostnameVerifier().toString());
+ } finally {
+ sslFactory.destroy();
+ }
+ }
+
}