You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2014/07/07 22:44:06 UTC
svn commit: r1608603 [1/3] - in
/hadoop/common/branches/fs-encryption/hadoop-common-project:
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/
hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/
hadoop-...
Author: wang
Date: Mon Jul 7 20:43:56 2014
New Revision: 1608603
URL: http://svn.apache.org/r1608603
Log:
Merge trunk to branch.
Added:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSExceptionMessages.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSExceptionMessages.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WeightedRoundRobinMultiplexer.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WeightedRoundRobinMultiplexer.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/markdown/
- copied from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/markdown/
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderDelegationTokenExtension.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderDelegationTokenExtension.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/
- copied from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestWeightedRoundRobinMultiplexer.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestWeightedRoundRobinMultiplexer.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCompositeGroupMapping.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCompositeGroupMapping.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java
- copied unchanged from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/resources/contract/
- copied from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/contract/
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/dev-support/
- copied from r1608600, hadoop/common/trunk/hadoop-common-project/hadoop-nfs/dev-support/
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverControllerStress.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSocketIOWithTimeout.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestAccessControlList.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSCacheKeyProvider.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/pom.xml
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SecurityHandler.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SysSecurityHandler.java
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java Mon Jul 7 20:43:56 2014
@@ -134,11 +134,15 @@ public class AuthenticationFilter implem
String authHandlerName = config.getProperty(AUTH_TYPE, null);
String authHandlerClassName;
if (authHandlerName == null) {
- throw new ServletException("Authentication type must be specified: simple|kerberos|<class>");
+ throw new ServletException("Authentication type must be specified: " +
+ PseudoAuthenticationHandler.TYPE + "|" +
+ KerberosAuthenticationHandler.TYPE + "|<class>");
}
- if (authHandlerName.equals("simple")) {
+ if (authHandlerName.toLowerCase(Locale.ENGLISH).equals(
+ PseudoAuthenticationHandler.TYPE)) {
authHandlerClassName = PseudoAuthenticationHandler.class.getName();
- } else if (authHandlerName.equals("kerberos")) {
+ } else if (authHandlerName.toLowerCase(Locale.ENGLISH).equals(
+ KerberosAuthenticationHandler.TYPE)) {
authHandlerClassName = KerberosAuthenticationHandler.class.getName();
} else {
authHandlerClassName = authHandlerName;
@@ -421,14 +425,20 @@ public class AuthenticationFilter implem
* cookie. It has no effect if its value < 0.
*
* XXX the following code duplicate some logic in Jetty / Servlet API,
- * because of the fact that Hadoop is stuck at servlet 3.0 and jetty 6
+ * because of the fact that Hadoop is stuck at servlet 2.5 and jetty 6
* right now.
*/
public static void createAuthCookie(HttpServletResponse resp, String token,
String domain, String path, long expires,
boolean isSecure) {
- StringBuilder sb = new StringBuilder(AuthenticatedURL.AUTH_COOKIE).append
- ("=").append(token);
+ StringBuilder sb = new StringBuilder(AuthenticatedURL.AUTH_COOKIE)
+ .append("=");
+ if (token != null && token.length() > 0) {
+ sb.append("\"")
+ .append(token)
+ .append("\"");
+ }
+ sb.append("; Version=1");
if (path != null) {
sb.append("; Path=").append(path);
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java Mon Jul 7 20:43:56 2014
@@ -74,6 +74,8 @@ public class TestAuthenticationFilter {
Assert.fail();
} catch (ServletException ex) {
// Expected
+ Assert.assertEquals("Authentication type must be specified: simple|kerberos|<class>",
+ ex.getMessage());
} catch (Exception ex) {
Assert.fail();
} finally {
@@ -233,6 +235,27 @@ public class TestAuthenticationFilter {
filter.destroy();
}
}
+
+ @Test
+ public void testInitCaseSensitivity() throws Exception {
+ // minimal configuration & simple auth handler (Pseudo)
+ AuthenticationFilter filter = new AuthenticationFilter();
+ try {
+ FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn("SimPle");
+ Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TOKEN_VALIDITY)).thenReturn(
+ (new Long(TOKEN_VALIDITY_SEC)).toString());
+ Mockito.when(config.getInitParameterNames()).thenReturn(
+ new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ AuthenticationFilter.AUTH_TOKEN_VALIDITY)).elements());
+
+ filter.init(config);
+ Assert.assertEquals(PseudoAuthenticationHandler.class,
+ filter.getAuthenticationHandler().getClass());
+ } finally {
+ filter.destroy();
+ }
+ }
@Test
public void testGetRequestURL() throws Exception {
@@ -508,21 +531,17 @@ public class TestAuthenticationFilter {
private static void parseCookieMap(String cookieHeader, HashMap<String,
String> cookieMap) {
- for (String pair : cookieHeader.split(";")) {
- String p = pair.trim();
- int idx = p.indexOf('=');
- final String k, v;
- if (idx == -1) {
- k = p;
- v = null;
- } else if (idx == p.length()) {
- k = p.substring(0, idx - 1);
- v = null;
- } else {
- k = p.substring(0, idx);
- v = p.substring(idx + 1);
+ List<HttpCookie> cookies = HttpCookie.parse(cookieHeader);
+ for (HttpCookie cookie : cookies) {
+ if (AuthenticatedURL.AUTH_COOKIE.equals(cookie.getName())) {
+ cookieMap.put(cookie.getName(), cookie.getValue());
+ if (cookie.getPath() != null) {
+ cookieMap.put("Path", cookie.getPath());
+ }
+ if (cookie.getDomain() != null) {
+ cookieMap.put("Domain", cookie.getDomain());
+ }
}
- cookieMap.put(k, v);
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt Mon Jul 7 20:43:56 2014
@@ -12,7 +12,20 @@ Trunk (Unreleased)
NEW FEATURES
HADOOP-10433. Key Management Server based on KeyProvider API. (tucu)
+
+ HADOOP-9629. Support Windows Azure Storage - Blob as a file system in Hadoop.
+ (Dexter Bradshaw, Mostafa Elhemali, Xi Fang, Johannes Klein, David Lao,
+ Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys,
+ Alexander Stojanovic, Brian Swan, and Min Wei via cnauroth)
+
+ HADOOP-10728. Metrics system for Windows Azure Storage Filesystem.
+ (Dexter Bradshaw, Mostafa Elhemali, Xi Fang, Johannes Klein, David Lao,
+ Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys,
+ Alexander Stojanovich, Brian Swan, and Min Wei via cnauroth)
+ HADOOP-10719. Add generateEncryptedKey and decryptEncryptedKey
+ methods to KeyProvider. (asuresh via tucu)
+
IMPROVEMENTS
HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution
@@ -152,6 +165,18 @@ Trunk (Unreleased)
HADOOP-10607. Create API to separate credential/password storage from
applications. (Larry McCay via omalley)
+ HADOOP-10696. Add optional attributes to KeyProvider Options and Metadata.
+ (tucu)
+
+ HADOOP-10695. KMSClientProvider should respect a configurable timeout.
+ (yoderme via tucu)
+
+ HADOOP-10757. KeyProvider KeyVersion should provide the key name.
+ (asuresh via tucu)
+
+ HADOOP-10769. Create KeyProvider extension to handle delegation tokens.
+ (Arun Suresh via atm)
+
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.
@@ -339,12 +364,34 @@ Trunk (Unreleased)
HADOOP-10611. KMS, keyVersion name should not be assumed to be
keyName@versionNumber. (tucu)
+ HADOOP-10717. HttpServer2 should load jsp DTD from local jars instead of
+ going remote. (Dapeng Sun via wheat9)
+
+ HADOOP-10689. InputStream is not closed in
+ AzureNativeFileSystemStore#retrieve(). (Chen He via cnauroth)
+
+ HADOOP-10690. Lack of synchronization on access to InputStream in
+ NativeAzureFileSystem#NativeAzureFsInputStream#close().
+ (Chen He via cnauroth)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
HADOOP-8589. ViewFs tests fail when tests and home dirs are nested (sanjay Radia)
+Release 2.6.0 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -355,10 +402,7 @@ Release 2.5.0 - UNRELEASED
HADOOP-9704. Write metrics sink plugin for Hadoop/Graphite (Chu Tong, Alex Newman and Babak Behzad via raviprak)
- HADOOP-9629. Support Windows Azure Storage - Blob as a file system in Hadoop.
- (Dexter Bradshaw, Mostafa Elhemali, Xi Fang, Johannes Klein, David Lao,
- Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys,
- Alexander Stojanovic, Brian Swan, and Min Wei via cnauroth)
+ HADOOP-8943. Support multiple group mapping providers. (Kai Zheng via brandonli)
IMPROVEMENTS
@@ -439,6 +483,48 @@ Release 2.5.0 - UNRELEASED
HADOOP-10557. FsShell -cp -pa option for preserving extended ACLs.
(Akira Ajisaka via cnauroth)
+ HADOOP-10279. Create multiplexer, a requirement for the fair queue.
+ (Chris Li via Arpit Agarwal)
+
+ HADOOP-10659. Refactor AccessControlList to reuse utility functions
+ and to improve performance. (Benoy Antony via Arpit Agarwal)
+
+ HADOOP-10665. Make Hadoop Authentication Handler loads case in-sensitive
+ (Benoy Antony via vinayakumarb)
+
+ HADOOP-10652. Refactor Proxyusers to use AccessControlList. (Benoy
+ Antony via Arpit Agarwal)
+
+ HADOOP-10747. Support configurable retries on SASL connection failures in
+ RPC client. (cnauroth)
+
+ HADOOP-10674. Improve PureJavaCrc32 performance and use java.util.zip.CRC32
+ for Java 7 and above. (szetszwo)
+
+ HADOOP-10754. Reenable several HA ZooKeeper-related tests on Windows.
+ (cnauroth)
+
+ HADOOP-10565. Support IP ranges (CIDR) in proxyuser.hosts. (Benoy Antony
+ via Arpit Agarwal)
+
+ HADOOP-10649. Allow overriding the default ACL for service authorization
+ (Benoy Antony via Arpit Agarwal)
+
+ HADOOP-10767. Clean up unused code in Ls shell command. (cnauroth)
+
+ HADOOP-9361 Strictly define the expected behavior of filesystem APIs and
+ write tests to verify compliance (stevel)
+
+ HADOOP-9651 Filesystems to throw FileAlreadyExistsException in
+ createFile(path, overwrite=false) when the file exists (stevel)
+
+ HADOOP-9495 Define behaviour of Seekable.seek(), write tests,
+ fix all hadoop implementations for compliance
+
+ HADOOP-10312 Shell.ExitCodeException to have more useful toString (stevel)
+
+ HADOOP-10782. Fix typo in DataChecksum class. (Jingguo Yao via suresh)
+
OPTIMIZATIONS
BUG FIXES
@@ -576,6 +662,23 @@ Release 2.5.0 - UNRELEASED
HADOOP-10716. Cannot use more than 1 har filesystem.
(Rushabh Shah via cnauroth)
+ HADOOP-9559. When metrics system is restarted MBean names get incorrectly
+ flagged as dupes. (Mostafa Elhemali and Mike Liddell via cnauroth)
+
+ HADOOP-10746. TestSocketIOWithTimeout#testSocketIOWithTimeout fails on
+ Power PC. (Jinghui Wang via Arpit Agarwal)
+
+ HADOOP-9705. FsShell cp -p does not preserve directory attibutes.
+ (Akira AJISAKA via cnauroth)
+
+ HADOOP-10739. Renaming a file into a directory containing the same
+ filename results in a confusing I/O error (chang li via jlowe)
+
+ HADOOP-10533 S3 input stream NPEs in MapReduce join (stevel)
+
+ HADOOP-10419 BufferedFSInputStream NPEs on getPos() on a closed stream
+ (stevel)
+
BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS
HADOOP-10520. Extended attributes definition and FileSystem APIs for
@@ -601,6 +704,14 @@ Release 2.5.0 - UNRELEASED
HADOOP-10711. Cleanup some extra dependencies from hadoop-auth. (rkanter via tucu)
+ HADOOP-10479. Fix new findbugs warnings in hadoop-minikdc.
+ (Swarnim Kulkarni via wheat9)
+
+ HADOOP-10715. Remove public GraphiteSink#setWriter (Babak Behzad via raviprak)
+
+ HADOOP-10710. hadoop.auth cookie is not properly constructed according to
+ RFC2109. (Juan Yu via tucu)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2.5/hadoop-common-project/hadoop-common/CHANGES.txt:r1607590,1607618
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1603979-1608600
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/pom.xml?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/pom.xml Mon Jul 7 20:43:56 2014
@@ -104,6 +104,11 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
<scope>compile</scope>
Propchange: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1602934-1608600
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html Mon Jul 7 20:43:56 2014
@@ -470,6 +470,10 @@ Tests run: 4, Failures: 0, Errors: 2, Sk
Major bug reported by Jian He and fixed by Vinod Kumar Vavilapalli <br>
<b>Few tests in TestJobClient fail on Windows</b><br>
<blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-5830">MAPREDUCE-5830</a>.
+ Blocker bug reported by Jason Lowe and fixed by Akira AJISAKA <br>
+ <b>HostUtil.getTaskLogUrl is not backwards binary compatible with 2.3</b><br>
+ <blockquote></blockquote></li>
<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-5828">MAPREDUCE-5828</a>.
Major bug reported by Vinod Kumar Vavilapalli and fixed by Vinod Kumar Vavilapalli <br>
<b>TestMapReduceJobControl fails on JDK 7 + Windows</b><br>
@@ -506,6 +510,10 @@ Tests run: 4, Failures: 0, Errors: 2, Sk
Trivial bug reported by Todd Lipcon and fixed by Chen He <br>
<b>docs for map output compression incorrectly reference SequenceFile</b><br>
<blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/HDFS-6527">HDFS-6527</a>.
+ Blocker bug reported by Kihwal Lee and fixed by Kihwal Lee <br>
+ <b>Edit log corruption due to defered INode removal</b><br>
+ <blockquote></blockquote></li>
<li> <a href="https://issues.apache.org/jira/browse/HDFS-6411">HDFS-6411</a>.
Major bug reported by Zhongyi Xie and fixed by Brandon Li (nfs)<br>
<b>nfs-hdfs-gateway mount raises I/O error and hangs when a unauthorized user attempts to access it</b><br>
Propchange: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1603664-1608600
Merged /hadoop/common/branches/branch-2.5/hadoop-common-project/hadoop-common/src/main/java:r1607590
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java Mon Jul 7 20:43:56 2014
@@ -173,7 +173,7 @@ public class JavaKeyStoreProvider extend
} catch (UnrecoverableKeyException e) {
throw new IOException("Can't recover key " + key + " from " + path, e);
}
- return new KeyVersion(versionName, key.getEncoded());
+ return new KeyVersion(getBaseName(versionName), versionName, key.getEncoded());
} finally {
readLock.unlock();
}
@@ -270,14 +270,14 @@ public class JavaKeyStoreProvider extend
e);
}
Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
- options.getDescription(), new Date(), 1);
+ options.getDescription(), options.getAttributes(), new Date(), 1);
if (options.getBitLength() != 8 * material.length) {
throw new IOException("Wrong key length. Required " +
options.getBitLength() + ", but got " + (8 * material.length));
}
cache.put(name, meta);
String versionName = buildVersionName(name, 0);
- return innerSetKeyVersion(versionName, material, meta.getCipher());
+ return innerSetKeyVersion(name, versionName, material, meta.getCipher());
} finally {
writeLock.unlock();
}
@@ -316,7 +316,7 @@ public class JavaKeyStoreProvider extend
}
}
- KeyVersion innerSetKeyVersion(String versionName, byte[] material,
+ KeyVersion innerSetKeyVersion(String name, String versionName, byte[] material,
String cipher) throws IOException {
try {
keyStore.setKeyEntry(versionName, new SecretKeySpec(material, cipher),
@@ -326,7 +326,7 @@ public class JavaKeyStoreProvider extend
e);
}
changed = true;
- return new KeyVersion(versionName, material);
+ return new KeyVersion(name, versionName, material);
}
@Override
@@ -344,7 +344,7 @@ public class JavaKeyStoreProvider extend
}
int nextVersion = meta.addVersion();
String versionName = buildVersionName(name, nextVersion);
- return innerSetKeyVersion(versionName, material, meta.getCipher());
+ return innerSetKeyVersion(name, versionName, material, meta.getCipher());
} finally {
writeLock.unlock();
}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java Mon Jul 7 20:43:56 2014
@@ -26,8 +26,11 @@ import java.io.OutputStreamWriter;
import java.net.URI;
import java.security.NoSuchAlgorithmException;
import java.text.MessageFormat;
+import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
@@ -60,14 +63,20 @@ public abstract class KeyProvider {
* The combination of both the key version name and the key material.
*/
public static class KeyVersion {
+ private final String name;
private final String versionName;
private final byte[] material;
- protected KeyVersion(String versionName,
+ protected KeyVersion(String name, String versionName,
byte[] material) {
+ this.name = name;
this.versionName = versionName;
this.material = material;
}
+
+ public String getName() {
+ return name;
+ }
public String getVersionName() {
return versionName;
@@ -107,18 +116,22 @@ public abstract class KeyProvider {
private final static String CREATED_FIELD = "created";
private final static String DESCRIPTION_FIELD = "description";
private final static String VERSIONS_FIELD = "versions";
+ private final static String ATTRIBUTES_FIELD = "attributes";
private final String cipher;
private final int bitLength;
private final String description;
private final Date created;
private int versions;
+ private Map<String, String> attributes;
- protected Metadata(String cipher, int bitLength,
- String description, Date created, int versions) {
+ protected Metadata(String cipher, int bitLength, String description,
+ Map<String, String> attributes, Date created, int versions) {
this.cipher = cipher;
this.bitLength = bitLength;
this.description = description;
+ this.attributes = (attributes == null || attributes.isEmpty())
+ ? null : attributes;
this.created = created;
this.versions = versions;
}
@@ -141,6 +154,11 @@ public abstract class KeyProvider {
return cipher;
}
+ @SuppressWarnings("unchecked")
+ public Map<String, String> getAttributes() {
+ return (attributes == null) ? Collections.EMPTY_MAP : attributes;
+ }
+
/**
* Get the algorithm from the cipher.
* @return the algorithm name
@@ -188,6 +206,13 @@ public abstract class KeyProvider {
if (description != null) {
writer.name(DESCRIPTION_FIELD).value(description);
}
+ if (attributes != null && attributes.size() > 0) {
+ writer.name(ATTRIBUTES_FIELD).beginObject();
+ for (Map.Entry<String, String> attribute : attributes.entrySet()) {
+ writer.name(attribute.getKey()).value(attribute.getValue());
+ }
+ writer.endObject();
+ }
writer.name(VERSIONS_FIELD).value(versions);
writer.endObject();
writer.flush();
@@ -208,6 +233,7 @@ public abstract class KeyProvider {
Date created = null;
int versions = 0;
String description = null;
+ Map<String, String> attributes = null;
JsonReader reader = new JsonReader(new InputStreamReader
(new ByteArrayInputStream(bytes)));
try {
@@ -224,6 +250,13 @@ public abstract class KeyProvider {
versions = reader.nextInt();
} else if (DESCRIPTION_FIELD.equals(field)) {
description = reader.nextString();
+ } else if (ATTRIBUTES_FIELD.equalsIgnoreCase(field)) {
+ reader.beginObject();
+ attributes = new HashMap<String, String>();
+ while (reader.hasNext()) {
+ attributes.put(reader.nextName(), reader.nextString());
+ }
+ reader.endObject();
}
}
reader.endObject();
@@ -234,6 +267,7 @@ public abstract class KeyProvider {
this.bitLength = bitLength;
this.created = created;
this.description = description;
+ this.attributes = attributes;
this.versions = versions;
}
}
@@ -245,6 +279,7 @@ public abstract class KeyProvider {
private String cipher;
private int bitLength;
private String description;
+ private Map<String, String> attributes;
public Options(Configuration conf) {
cipher = conf.get(DEFAULT_CIPHER_NAME, DEFAULT_CIPHER);
@@ -266,6 +301,16 @@ public abstract class KeyProvider {
return this;
}
+ public Options setAttributes(Map<String, String> attributes) {
+ if (attributes != null) {
+ if (attributes.containsKey(null)) {
+ throw new IllegalArgumentException("attributes cannot have a NULL key");
+ }
+ this.attributes = new HashMap<String, String>(attributes);
+ }
+ return this;
+ }
+
public String getCipher() {
return cipher;
}
@@ -277,6 +322,11 @@ public abstract class KeyProvider {
public String getDescription() {
return description;
}
+
+ @SuppressWarnings("unchecked")
+ public Map<String, String> getAttributes() {
+ return (attributes == null) ? Collections.EMPTY_MAP : attributes;
+ }
}
/**
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java Mon Jul 7 20:43:56 2014
@@ -55,12 +55,13 @@ public class UserProvider extends KeyPro
}
@Override
- public synchronized KeyVersion getKeyVersion(String versionName) {
+ public synchronized KeyVersion getKeyVersion(String versionName)
+ throws IOException {
byte[] bytes = credentials.getSecretKey(new Text(versionName));
if (bytes == null) {
return null;
}
- return new KeyVersion(versionName, bytes);
+ return new KeyVersion(getBaseName(versionName), versionName, bytes);
}
@Override
@@ -89,12 +90,12 @@ public class UserProvider extends KeyPro
options.getBitLength() + ", but got " + (8 * material.length));
}
Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
- options.getDescription(), new Date(), 1);
+ options.getDescription(), options.getAttributes(), new Date(), 1);
cache.put(name, meta);
String versionName = buildVersionName(name, 0);
credentials.addSecretKey(nameT, meta.serialize());
credentials.addSecretKey(new Text(versionName), material);
- return new KeyVersion(versionName, material);
+ return new KeyVersion(name, versionName, material);
}
@Override
@@ -125,7 +126,7 @@ public class UserProvider extends KeyPro
credentials.addSecretKey(new Text(name), meta.serialize());
String versionName = buildVersionName(name, nextVersion);
credentials.addSecretKey(new Text(versionName), material);
- return new KeyVersion(versionName, material);
+ return new KeyVersion(name, versionName, material);
}
@Override
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java Mon Jul 7 20:43:56 2014
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.crypto.key.kms;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -27,6 +26,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.http.client.utils.URIBuilder;
@@ -71,18 +71,27 @@ public class KMSClientProvider extends K
private static final String HTTP_PUT = "PUT";
private static final String HTTP_DELETE = "DELETE";
+
+ private static final String CONFIG_PREFIX = "hadoop.security.kms.client.";
+
+ /* It's possible to specify a timeout, in seconds, in the config file */
+ public static final String TIMEOUT_ATTR = CONFIG_PREFIX + "timeout";
+ public static final int DEFAULT_TIMEOUT = 60;
+
private static KeyVersion parseJSONKeyVersion(Map valueMap) {
KeyVersion keyVersion = null;
if (!valueMap.isEmpty()) {
byte[] material = (valueMap.containsKey(KMSRESTConstants.MATERIAL_FIELD))
? Base64.decodeBase64((String) valueMap.get(KMSRESTConstants.MATERIAL_FIELD))
: null;
- keyVersion = new KMSKeyVersion((String)
- valueMap.get(KMSRESTConstants.VERSION_NAME_FIELD), material);
+ String versionName = (String)valueMap.get(KMSRESTConstants.VERSION_NAME_FIELD);
+ String keyName = (String)valueMap.get(KMSRESTConstants.NAME_FIELD);
+ keyVersion = new KMSKeyVersion(keyName, versionName, material);
}
return keyVersion;
}
+ @SuppressWarnings("unchecked")
private static Metadata parseJSONMetadata(Map valueMap) {
Metadata metadata = null;
if (!valueMap.isEmpty()) {
@@ -90,6 +99,7 @@ public class KMSClientProvider extends K
(String) valueMap.get(KMSRESTConstants.CIPHER_FIELD),
(Integer) valueMap.get(KMSRESTConstants.LENGTH_FIELD),
(String) valueMap.get(KMSRESTConstants.DESCRIPTION_FIELD),
+ (Map<String, String>) valueMap.get(KMSRESTConstants.ATTRIBUTES_FIELD),
new Date((Long) valueMap.get(KMSRESTConstants.CREATED_FIELD)),
(Integer) valueMap.get(KMSRESTConstants.VERSIONS_FIELD));
}
@@ -139,6 +149,7 @@ public class KMSClientProvider extends K
private String kmsUrl;
private SSLFactory sslFactory;
+ private ConnectionConfigurator configurator;
@Override
public String toString() {
@@ -147,6 +158,42 @@ public class KMSClientProvider extends K
return sb.toString();
}
+ /**
+ * This small class exists to set the timeout values for a connection
+ */
+ private static class TimeoutConnConfigurator
+ implements ConnectionConfigurator {
+ private ConnectionConfigurator cc;
+ private int timeout;
+
+ /**
+ * Sets the timeout and wraps another connection configurator
+ * @param timeout - will set both connect and read timeouts - in seconds
+ * @param cc - another configurator to wrap - may be null
+ */
+ public TimeoutConnConfigurator(int timeout, ConnectionConfigurator cc) {
+ this.timeout = timeout;
+ this.cc = cc;
+ }
+
+ /**
+ * Calls the wrapped configure() method, then sets timeouts
+ * @param conn the {@link HttpURLConnection} instance to configure.
+ * @return the connection
+ * @throws IOException
+ */
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ if (cc != null) {
+ conn = cc.configure(conn);
+ }
+ conn.setConnectTimeout(timeout * 1000); // conversion to milliseconds
+ conn.setReadTimeout(timeout * 1000);
+ return conn;
+ }
+ }
+
public KMSClientProvider(URI uri, Configuration conf) throws IOException {
Path path = ProviderUtils.unnestUri(uri);
URL url = path.toUri().toURL();
@@ -159,6 +206,8 @@ public class KMSClientProvider extends K
throw new IOException(ex);
}
}
+ int timeout = conf.getInt(TIMEOUT_ATTR, DEFAULT_TIMEOUT);
+ configurator = new TimeoutConnConfigurator(timeout, sslFactory);
}
private String createServiceURL(URL url) throws IOException {
@@ -220,7 +269,7 @@ public class KMSClientProvider extends K
HttpURLConnection conn;
try {
AuthenticatedURL authUrl = new AuthenticatedURL(new PseudoAuthenticator(),
- sslFactory);
+ configurator);
conn = authUrl.openConnection(url, new AuthenticatedURL.Token());
} catch (AuthenticationException ex) {
throw new IOException(ex);
@@ -314,8 +363,8 @@ public class KMSClientProvider extends K
}
public static class KMSKeyVersion extends KeyVersion {
- public KMSKeyVersion(String versionName, byte[] material) {
- super(versionName, material);
+ public KMSKeyVersion(String keyName, String versionName, byte[] material) {
+ super(keyName, versionName, material);
}
}
@@ -351,8 +400,8 @@ public class KMSClientProvider extends K
public static class KMSMetadata extends Metadata {
public KMSMetadata(String cipher, int bitLength, String description,
- Date created, int versions) {
- super(cipher, bitLength, description, created, versions);
+ Map<String, String> attributes, Date created, int versions) {
+ super(cipher, bitLength, description, attributes, created, versions);
}
}
@@ -416,6 +465,9 @@ public class KMSClientProvider extends K
jsonKey.put(KMSRESTConstants.DESCRIPTION_FIELD,
options.getDescription());
}
+ if (options.getAttributes() != null && !options.getAttributes().isEmpty()) {
+ jsonKey.put(KMSRESTConstants.ATTRIBUTES_FIELD, options.getAttributes());
+ }
URL url = createURL(KMSRESTConstants.KEYS_RESOURCE, null, null, null);
HttpURLConnection conn = createConnection(url, HTTP_POST);
conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME);
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java Mon Jul 7 20:43:56 2014
@@ -42,6 +42,7 @@ public class KMSRESTConstants {
public static final String CIPHER_FIELD = "cipher";
public static final String LENGTH_FIELD = "length";
public static final String DESCRIPTION_FIELD = "description";
+ public static final String ATTRIBUTES_FIELD = "attributes";
public static final String CREATED_FIELD = "created";
public static final String VERSIONS_FIELD = "versions";
public static final String MATERIAL_FIELD = "material";
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java Mon Jul 7 20:43:56 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs;
import java.io.BufferedInputStream;
+import java.io.EOFException;
import java.io.FileDescriptor;
import java.io.IOException;
@@ -51,6 +52,9 @@ implements Seekable, PositionedReadable,
@Override
public long getPos() throws IOException {
+ if (in == null) {
+ throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+ }
return ((FSInputStream)in).getPos()-(count-pos);
}
@@ -66,8 +70,11 @@ implements Seekable, PositionedReadable,
@Override
public void seek(long pos) throws IOException {
- if( pos<0 ) {
- return;
+ if (in == null) {
+ throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+ }
+ if (pos < 0) {
+ throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
}
if (this.pos != this.count) {
// optimize: check if the pos is in the buffer
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java Mon Jul 7 20:43:56 2014
@@ -18,7 +18,10 @@
package org.apache.hadoop.fs;
-import java.io.*;
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
import java.nio.channels.ClosedChannelException;
import java.util.Arrays;
@@ -26,8 +29,8 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.PureJavaCrc32;
/****************************************************************
* Abstract Checksumed FileSystem.
@@ -147,7 +150,7 @@ public abstract class ChecksumFileSystem
if (!Arrays.equals(version, CHECKSUM_VERSION))
throw new IOException("Not a checksum file: "+sumFile);
this.bytesPerSum = sums.readInt();
- set(fs.verifyChecksum, new PureJavaCrc32(), bytesPerSum, 4);
+ set(fs.verifyChecksum, DataChecksum.newCrc32(), bytesPerSum, 4);
} catch (FileNotFoundException e) { // quietly ignore
set(fs.verifyChecksum, null, 1, 0);
} catch (IOException e) { // loudly ignore
@@ -259,8 +262,7 @@ public abstract class ChecksumFileSystem
private Path file;
private long fileLen = -1L;
- FSDataBoundedInputStream(FileSystem fs, Path file, InputStream in)
- throws IOException {
+ FSDataBoundedInputStream(FileSystem fs, Path file, InputStream in) {
super(in);
this.fs = fs;
this.file = file;
@@ -317,8 +319,8 @@ public abstract class ChecksumFileSystem
@Override
public synchronized void seek(long pos) throws IOException {
- if(pos>getFileLength()) {
- throw new IOException("Cannot seek after EOF");
+ if (pos > getFileLength()) {
+ throw new EOFException("Cannot seek after EOF");
}
super.seek(pos);
}
@@ -379,7 +381,7 @@ public abstract class ChecksumFileSystem
long blockSize,
Progressable progress)
throws IOException {
- super(new PureJavaCrc32(), fs.getBytesPerSum(), 4);
+ super(DataChecksum.newCrc32(), fs.getBytesPerSum(), 4);
int bytesPerSum = fs.getBytesPerSum();
this.datas = fs.getRawFileSystem().create(file, overwrite, bufferSize,
replication, blockSize, progress);
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java Mon Jul 7 20:43:56 2014
@@ -18,7 +18,9 @@
package org.apache.hadoop.fs;
-import java.io.*;
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
@@ -31,8 +33,8 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.PureJavaCrc32;
/**
* Abstract Checksumed Fs.
@@ -139,7 +141,7 @@ public abstract class ChecksumFs extends
throw new IOException("Not a checksum file: "+sumFile);
}
this.bytesPerSum = sums.readInt();
- set(fs.verifyChecksum, new PureJavaCrc32(), bytesPerSum, 4);
+ set(fs.verifyChecksum, DataChecksum.newCrc32(), bytesPerSum, 4);
} catch (FileNotFoundException e) { // quietly ignore
set(fs.verifyChecksum, null, 1, 0);
} catch (IOException e) { // loudly ignore
@@ -335,7 +337,7 @@ public abstract class ChecksumFs extends
final short replication, final long blockSize,
final Progressable progress, final ChecksumOpt checksumOpt,
final boolean createParent) throws IOException {
- super(new PureJavaCrc32(), fs.getBytesPerSum(), 4);
+ super(DataChecksum.newCrc32(), fs.getBytesPerSum(), 4);
// checksumOpt is passed down to the raw fs. Unless it implements
// checksum impelemts internally, checksumOpt will be ignored.
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Mon Jul 7 20:43:56 2014
@@ -131,6 +131,9 @@ public class CommonConfigurationKeys ext
* Service Authorization
*/
public static final String
+ HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_ACL =
+ "security.service.authorization.default.acl";
+ public static final String
HADOOP_SECURITY_SERVICE_AUTHORIZATION_REFRESH_POLICY =
"security.refresh.policy.protocol.acl";
public static final String
@@ -253,6 +256,10 @@ public class CommonConfigurationKeys ext
public static final String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY = "ipc.client.fallback-to-simple-auth-allowed";
public static final boolean IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT = false;
+ public static final String IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY =
+ "ipc.client.connect.max.retries.on.sasl";
+ public static final int IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_DEFAULT = 5;
+
/** How often the server scans for idle connections */
public static final String IPC_CLIENT_CONNECTION_IDLESCANINTERVAL_KEY =
"ipc.client.connection.idle-scan-interval.ms";
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java Mon Jul 7 20:43:56 2014
@@ -67,7 +67,10 @@ public class FSDataOutputStream extends
@Override
public void close() throws IOException {
- out.close();
+ // ensure close works even if a null reference was passed in
+ if (out != null) {
+ out.close();
+ }
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java Mon Jul 7 20:43:56 2014
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs;
+import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.Checksum;
@@ -394,8 +395,8 @@ abstract public class FSInputChecker ext
@Override
public synchronized void seek(long pos) throws IOException {
- if( pos<0 ) {
- return;
+ if( pos < 0 ) {
+ throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
}
// optimize: check if the pos is in the buffer
long start = chunkPos - this.count;
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java Mon Jul 7 20:43:56 2014
@@ -23,6 +23,7 @@ import com.google.common.annotations.Vis
import java.io.BufferedOutputStream;
import java.io.DataOutput;
+import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
@@ -105,6 +106,10 @@ public class RawLocalFileSystem extends
@Override
public void seek(long pos) throws IOException {
+ if (pos < 0) {
+ throw new EOFException(
+ FSExceptionMessages.NEGATIVE_SEEK);
+ }
fis.getChannel().position(pos);
this.position = pos;
}
@@ -256,7 +261,7 @@ public class RawLocalFileSystem extends
boolean createParent, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
if (exists(f) && !overwrite) {
- throw new IOException("File already exists: "+f);
+ throw new FileAlreadyExistsException("File already exists: " + f);
}
Path parent = f.getParent();
if (parent != null && !mkdirs(parent)) {
@@ -272,7 +277,7 @@ public class RawLocalFileSystem extends
EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
if (exists(f) && !flags.contains(CreateFlag.OVERWRITE)) {
- throw new IOException("File already exists: "+f);
+ throw new FileAlreadyExistsException("File already exists: " + f);
}
return new FSDataOutputStream(new BufferedOutputStream(
new LocalFSFileOutputStream(f, false), bufferSize), statistics);
@@ -344,6 +349,10 @@ public class RawLocalFileSystem extends
@Override
public boolean delete(Path p, boolean recursive) throws IOException {
File f = pathToFile(p);
+ if (!f.exists()) {
+ //no path, return false "nothing to delete"
+ return false;
+ }
if (f.isFile()) {
return f.delete();
} else if (!recursive && f.isDirectory() &&
@@ -412,10 +421,14 @@ public class RawLocalFileSystem extends
if(parent != null) {
File parent2f = pathToFile(parent);
if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) {
- throw new FileAlreadyExistsException("Parent path is not a directory: "
+ throw new ParentNotDirectoryException("Parent path is not a directory: "
+ parent);
}
}
+ if (p2f.exists() && !p2f.isDirectory()) {
+ throw new FileNotFoundException("Destination exists" +
+ " and is not a directory: " + p2f.getCanonicalPath());
+ }
return (parent == null || mkdirs(parent)) &&
(p2f.mkdir() || p2f.isDirectory());
}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java Mon Jul 7 20:43:56 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.ftp;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.net.ConnectException;
import java.net.URI;
import org.apache.commons.logging.Log;
@@ -33,11 +34,14 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Progressable;
/**
@@ -56,6 +60,12 @@ public class FTPFileSystem extends FileS
public static final int DEFAULT_BUFFER_SIZE = 1024 * 1024;
public static final int DEFAULT_BLOCK_SIZE = 4 * 1024;
+ public static final String FS_FTP_USER_PREFIX = "fs.ftp.user.";
+ public static final String FS_FTP_HOST = "fs.ftp.host";
+ public static final String FS_FTP_HOST_PORT = "fs.ftp.host.port";
+ public static final String FS_FTP_PASSWORD_PREFIX = "fs.ftp.password.";
+ public static final String E_SAME_DIRECTORY_ONLY =
+ "only same directory renames are supported";
private URI uri;
@@ -75,11 +85,11 @@ public class FTPFileSystem extends FileS
super.initialize(uri, conf);
// get host information from uri (overrides info in conf)
String host = uri.getHost();
- host = (host == null) ? conf.get("fs.ftp.host", null) : host;
+ host = (host == null) ? conf.get(FS_FTP_HOST, null) : host;
if (host == null) {
throw new IOException("Invalid host specified");
}
- conf.set("fs.ftp.host", host);
+ conf.set(FS_FTP_HOST, host);
// get port information from uri, (overrides info in conf)
int port = uri.getPort();
@@ -96,11 +106,11 @@ public class FTPFileSystem extends FileS
}
}
String[] userPasswdInfo = userAndPassword.split(":");
- conf.set("fs.ftp.user." + host, userPasswdInfo[0]);
+ conf.set(FS_FTP_USER_PREFIX + host, userPasswdInfo[0]);
if (userPasswdInfo.length > 1) {
- conf.set("fs.ftp.password." + host, userPasswdInfo[1]);
+ conf.set(FS_FTP_PASSWORD_PREFIX + host, userPasswdInfo[1]);
} else {
- conf.set("fs.ftp.password." + host, null);
+ conf.set(FS_FTP_PASSWORD_PREFIX + host, null);
}
setConf(conf);
this.uri = uri;
@@ -115,23 +125,24 @@ public class FTPFileSystem extends FileS
private FTPClient connect() throws IOException {
FTPClient client = null;
Configuration conf = getConf();
- String host = conf.get("fs.ftp.host");
- int port = conf.getInt("fs.ftp.host.port", FTP.DEFAULT_PORT);
- String user = conf.get("fs.ftp.user." + host);
- String password = conf.get("fs.ftp.password." + host);
+ String host = conf.get(FS_FTP_HOST);
+ int port = conf.getInt(FS_FTP_HOST_PORT, FTP.DEFAULT_PORT);
+ String user = conf.get(FS_FTP_USER_PREFIX + host);
+ String password = conf.get(FS_FTP_PASSWORD_PREFIX + host);
client = new FTPClient();
client.connect(host, port);
int reply = client.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
- throw new IOException("Server - " + host
- + " refused connection on port - " + port);
+ throw NetUtils.wrapException(host, port,
+ NetUtils.UNKNOWN_HOST, 0,
+ new ConnectException("Server response " + reply));
} else if (client.login(user, password)) {
client.setFileTransferMode(FTP.BLOCK_TRANSFER_MODE);
client.setFileType(FTP.BINARY_FILE_TYPE);
client.setBufferSize(DEFAULT_BUFFER_SIZE);
} else {
throw new IOException("Login failed on server - " + host + ", port - "
- + port);
+ + port + " as user '" + user + "'");
}
return client;
@@ -179,7 +190,7 @@ public class FTPFileSystem extends FileS
FileStatus fileStat = getFileStatus(client, absolute);
if (fileStat.isDirectory()) {
disconnect(client);
- throw new IOException("Path " + file + " is a directory.");
+ throw new FileNotFoundException("Path " + file + " is a directory.");
}
client.allocate(bufferSize);
Path parent = absolute.getParent();
@@ -214,12 +225,18 @@ public class FTPFileSystem extends FileS
final FTPClient client = connect();
Path workDir = new Path(client.printWorkingDirectory());
Path absolute = makeAbsolute(workDir, file);
- if (exists(client, file)) {
- if (overwrite) {
- delete(client, file);
+ FileStatus status;
+ try {
+ status = getFileStatus(client, file);
+ } catch (FileNotFoundException fnfe) {
+ status = null;
+ }
+ if (status != null) {
+ if (overwrite && !status.isDirectory()) {
+ delete(client, file, false);
} else {
disconnect(client);
- throw new IOException("File already exists: " + file);
+ throw new FileAlreadyExistsException("File already exists: " + file);
}
}
@@ -272,14 +289,13 @@ public class FTPFileSystem extends FileS
* Convenience method, so that we don't open a new connection when using this
* method from within another method. Otherwise every API invocation incurs
* the overhead of opening/closing a TCP connection.
+ * @throws IOException on IO problems other than FileNotFoundException
*/
- private boolean exists(FTPClient client, Path file) {
+ private boolean exists(FTPClient client, Path file) throws IOException {
try {
return getFileStatus(client, file) != null;
} catch (FileNotFoundException fnfe) {
return false;
- } catch (IOException ioe) {
- throw new FTPException("Failed to get file status", ioe);
}
}
@@ -294,12 +310,6 @@ public class FTPFileSystem extends FileS
}
}
- /** @deprecated Use delete(Path, boolean) instead */
- @Deprecated
- private boolean delete(FTPClient client, Path file) throws IOException {
- return delete(client, file, false);
- }
-
/**
* Convenience method, so that we don't open a new connection when using this
* method from within another method. Otherwise every API invocation incurs
@@ -310,9 +320,14 @@ public class FTPFileSystem extends FileS
Path workDir = new Path(client.printWorkingDirectory());
Path absolute = makeAbsolute(workDir, file);
String pathName = absolute.toUri().getPath();
- FileStatus fileStat = getFileStatus(client, absolute);
- if (fileStat.isFile()) {
- return client.deleteFile(pathName);
+ try {
+ FileStatus fileStat = getFileStatus(client, absolute);
+ if (fileStat.isFile()) {
+ return client.deleteFile(pathName);
+ }
+ } catch (FileNotFoundException e) {
+ //the file is not there
+ return false;
}
FileStatus[] dirEntries = listStatus(client, absolute);
if (dirEntries != null && dirEntries.length > 0 && !(recursive)) {
@@ -491,7 +506,7 @@ public class FTPFileSystem extends FileS
created = created && client.makeDirectory(pathName);
}
} else if (isFile(client, absolute)) {
- throw new IOException(String.format(
+ throw new ParentNotDirectoryException(String.format(
"Can't make directory for path %s since it is a file.", absolute));
}
return created;
@@ -528,6 +543,23 @@ public class FTPFileSystem extends FileS
}
/**
+ * Probe for a path being a parent of another
+ * @param parent parent path
+ * @param child possible child path
+ * @return true if the parent's path matches the start of the child's
+ */
+ private boolean isParentOf(Path parent, Path child) {
+ URI parentURI = parent.toUri();
+ String parentPath = parentURI.getPath();
+ if (!parentPath.endsWith("/")) {
+ parentPath += "/";
+ }
+ URI childURI = child.toUri();
+ String childPath = childURI.getPath();
+ return childPath.startsWith(parentPath);
+ }
+
+ /**
* Convenience method, so that we don't open a new connection when using this
* method from within another method. Otherwise every API invocation incurs
* the overhead of opening/closing a TCP connection.
@@ -544,20 +576,31 @@ public class FTPFileSystem extends FileS
Path absoluteSrc = makeAbsolute(workDir, src);
Path absoluteDst = makeAbsolute(workDir, dst);
if (!exists(client, absoluteSrc)) {
- throw new IOException("Source path " + src + " does not exist");
+ throw new FileNotFoundException("Source path " + src + " does not exist");
+ }
+ if (isDirectory(absoluteDst)) {
+ // destination is a directory: rename goes underneath it with the
+ // source name
+ absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
}
if (exists(client, absoluteDst)) {
- throw new IOException("Destination path " + dst
- + " already exist, cannot rename!");
+ throw new FileAlreadyExistsException("Destination path " + dst
+ + " already exists");
}
String parentSrc = absoluteSrc.getParent().toUri().toString();
String parentDst = absoluteDst.getParent().toUri().toString();
- String from = src.getName();
- String to = dst.getName();
+ if (isParentOf(absoluteSrc, absoluteDst)) {
+ throw new IOException("Cannot rename " + absoluteSrc + " under itself"
+ + " : "+ absoluteDst);
+ }
+
if (!parentSrc.equals(parentDst)) {
- throw new IOException("Cannot rename parent(source): " + parentSrc
- + ", parent(destination): " + parentDst);
+ throw new IOException("Cannot rename source: " + absoluteSrc
+ + " to " + absoluteDst
+ + " -"+ E_SAME_DIRECTORY_ONLY);
}
+ String from = absoluteSrc.getName();
+ String to = absoluteDst.getName();
client.changeWorkingDirectory(parentSrc);
boolean renamed = client.rename(from, to);
return renamed;
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java Mon Jul 7 20:43:56 2014
@@ -103,7 +103,7 @@ public class FTPInputStream extends FSIn
@Override
public synchronized void close() throws IOException {
if (closed) {
- throw new IOException("Stream closed");
+ return;
}
super.close();
closed = true;
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java Mon Jul 7 20:43:56 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -226,7 +227,7 @@ public class S3FileSystem extends FileSy
if (overwrite) {
delete(file, true);
} else {
- throw new IOException("File already exists: " + file);
+ throw new FileAlreadyExistsException("File already exists: " + file);
}
} else {
Path parent = file.getParent();