You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2014/09/11 09:24:41 UTC
[01/20] git commit: HDFS-6776. Using distcp to copy data between
insecure and secure cluster via webdhfs doesn't work. (yzhangal via tucu)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6584 db41a1b7b -> 70dfe9cfa
HDFS-6776. Using distcp to copy data between insecure and secure cluster via webdhfs doesn't work. (yzhangal via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbff44cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbff44cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbff44cb
Branch: refs/heads/HDFS-6584
Commit: bbff44cb03d0150f990acc3b77170893241cc282
Parents: 6dae4b4
Author: Alejandro Abdelnur <tu...@apache.org>
Authored: Tue Sep 9 22:16:42 2014 -0700
Committer: Alejandro Abdelnur <tu...@apache.org>
Committed: Tue Sep 9 22:16:42 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../DelegationTokenSecretManager.java | 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 3 ++
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 18 ++++++++-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 41 ++++++++++++++++++++
5 files changed, 65 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff44cb/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0b914ac..fa00d44 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -739,6 +739,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6986. DistributedFileSystem must get delegation tokens from configured
KeyProvider. (zhz via tucu)
+ HDFS-6776. Using distcp to copy data between insecure and secure cluster via webdhfs
+ doesn't work. (yzhangal via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff44cb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
index 175e3ed..8af7eba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
@@ -402,8 +402,7 @@ public class DelegationTokenSecretManager
final Token<DelegationTokenIdentifier> token = namenode.getRpcServer(
).getDelegationToken(new Text(renewer));
if (token == null) {
- throw new IOException("Failed to get the token for " + renewer
- + ", user=" + ugi.getShortUserName());
+ return null;
}
final InetSocketAddress addr = namenode.getNameNodeAddress();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff44cb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index 991885b..3949fbd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -283,6 +283,9 @@ public class NamenodeWebHdfsMethods {
final String renewer) throws IOException {
final Credentials c = DelegationTokenSecretManager.createCredentials(
namenode, ugi, renewer != null? renewer: ugi.getShortUserName());
+ if (c == null) {
+ return null;
+ }
final Token<? extends TokenIdentifier> t = c.getAllTokens().iterator().next();
Text kind = request.getScheme().equals("http") ? WebHdfsFileSystem.TOKEN_KIND
: SWebHdfsFileSystem.TOKEN_KIND;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff44cb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index cf6233f..40312ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -41,6 +41,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.DelegationTokenRenewer;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -102,6 +103,11 @@ public class WebHdfsFileSystem extends FileSystem
/** Delegation token kind */
public static final Text TOKEN_KIND = new Text("WEBHDFS delegation");
+
+ @VisibleForTesting
+ public static final String CANT_FALLBACK_TO_INSECURE_MSG =
+ "The client is configured to only allow connecting to secure cluster";
+
private boolean canRefreshDelegationToken;
private UserGroupInformation ugi;
@@ -112,6 +118,7 @@ public class WebHdfsFileSystem extends FileSystem
private Path workingDir;
private InetSocketAddress nnAddrs[];
private int currentNNAddrIndex;
+ private boolean disallowFallbackToInsecureCluster;
/**
* Return the protocol scheme for the FileSystem.
@@ -194,6 +201,9 @@ public class WebHdfsFileSystem extends FileSystem
this.workingDir = getHomeDirectory();
this.canRefreshDelegationToken = UserGroupInformation.isSecurityEnabled();
+ this.disallowFallbackToInsecureCluster = !conf.getBoolean(
+ CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT);
this.delegationToken = null;
}
@@ -1293,7 +1303,13 @@ public class WebHdfsFileSystem extends FileSystem
return JsonUtil.toDelegationToken(json);
}
}.run();
- token.setService(tokenServiceName);
+ if (token != null) {
+ token.setService(tokenServiceName);
+ } else {
+ if (disallowFallbackToInsecureCluster) {
+ throw new AccessControlException(CANT_FALLBACK_TO_INSECURE_MSG);
+ }
+ }
return token;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff44cb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 1431211..eec49d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -29,6 +29,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -45,6 +46,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.ipc.RetriableException;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
@@ -482,4 +484,43 @@ public class TestWebHDFS {
}
}
}
+
+ @Test
+ public void testDTInInsecureClusterWithFallback()
+ throws IOException, URISyntaxException {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ conf.setBoolean(CommonConfigurationKeys
+ .IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY, true);
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ final FileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
+ WebHdfsFileSystem.SCHEME);
+ Assert.assertNull(webHdfs.getDelegationToken(null));
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testDTInInsecureCluster() throws Exception {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ final FileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
+ WebHdfsFileSystem.SCHEME);
+ webHdfs.getDelegationToken(null);
+ fail("No exception is thrown.");
+ } catch (AccessControlException ace) {
+ Assert.assertTrue(ace.getMessage().startsWith(
+ WebHdfsFileSystem.CANT_FALLBACK_TO_INSECURE_MSG));
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
[02/20] git commit: HADOOP-11077. NPE if hosts not specified in
ProxyUsers. (gchanan via tucu)
Posted by sz...@apache.org.
HADOOP-11077. NPE if hosts not specified in ProxyUsers. (gchanan via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ee891aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ee891aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ee891aa
Branch: refs/heads/HDFS-6584
Commit: 9ee891aa90333bf18cba412400daa5834f15c41d
Parents: bbff44c
Author: Alejandro Abdelnur <tu...@apache.org>
Authored: Tue Sep 9 22:18:03 2014 -0700
Committer: Alejandro Abdelnur <tu...@apache.org>
Committed: Tue Sep 9 22:18:03 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../authorize/DefaultImpersonationProvider.java | 2 +-
.../hadoop/security/authorize/TestProxyUsers.java | 15 +++++++++++++++
3 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ee891aa/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index c60a9b7..b015087 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -777,6 +777,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10925. Compilation fails in native link0 function on Windows.
(cnauroth)
+ HADOOP-11077. NPE if hosts not specified in ProxyUsers. (gchanan via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ee891aa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
index ab1c390..b36ac80 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
@@ -123,7 +123,7 @@ public class DefaultImpersonationProvider implements ImpersonationProvider {
MachineList MachineList = proxyHosts.get(
getProxySuperuserIpConfKey(realUser.getShortUserName()));
- if(!MachineList.includes(remoteAddress)) {
+ if(MachineList == null || !MachineList.includes(remoteAddress)) {
throw new AuthorizationException("Unauthorized connection for super-user: "
+ realUser.getUserName() + " from IP " + remoteAddress);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ee891aa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
index dbcac67..8ff4bfb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
@@ -478,6 +478,21 @@ public class TestProxyUsers {
assertNotAuthorized(proxyUserUgi, "1.2.3.5");
}
+ @Test
+ public void testNoHostsForUsers() throws Exception {
+ Configuration conf = new Configuration(false);
+ conf.set("y." + REAL_USER_NAME + ".users",
+ StringUtils.join(",", Arrays.asList(AUTHORIZED_PROXY_USER_NAME)));
+ ProxyUsers.refreshSuperUserGroupsConfiguration(conf, "y");
+
+ UserGroupInformation realUserUgi = UserGroupInformation
+ .createRemoteUser(REAL_USER_NAME);
+ UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting(
+ AUTHORIZED_PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
+
+ // IP doesn't matter
+ assertNotAuthorized(proxyUserUgi, "1.2.3.4");
+ }
private void assertNotAuthorized(UserGroupInformation proxyUgi, String host) {
try {
[14/20] HADOOP-11074. Move s3-related FS connector code to
hadoop-aws. (David S. Wang via Colin Patrick McCabe)
Posted by sz...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestInMemoryS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestInMemoryS3FileSystemContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestInMemoryS3FileSystemContract.java
deleted file mode 100644
index 5d66cf1..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestInMemoryS3FileSystemContract.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.IOException;
-
-public class TestInMemoryS3FileSystemContract
- extends S3FileSystemContractBaseTest {
-
- @Override
- FileSystemStore getFileSystemStore() throws IOException {
- return new InMemoryFileSystemStore();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestS3Credentials.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestS3Credentials.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestS3Credentials.java
deleted file mode 100644
index bcbf0dc..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestS3Credentials.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3;
-
-import java.net.URI;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class TestS3Credentials extends TestCase {
- public void testInvalidHostnameWithUnderscores() throws Exception {
- S3Credentials s3Credentials = new S3Credentials();
- try {
- s3Credentials.initialize(new URI("s3://a:b@c_d"), new Configuration());
- fail("Should throw IllegalArgumentException");
- } catch (IllegalArgumentException e) {
- assertEquals("Invalid hostname in URI s3://a:b@c_d", e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestS3FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestS3FileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestS3FileSystem.java
deleted file mode 100644
index f21989c..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestS3FileSystem.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.IOException;
-import java.net.URI;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class TestS3FileSystem extends TestCase {
-
- public void testInitialization() throws IOException {
- initializationTest("s3://a:b@c", "s3://a:b@c");
- initializationTest("s3://a:b@c/", "s3://a:b@c");
- initializationTest("s3://a:b@c/path", "s3://a:b@c");
- initializationTest("s3://a@c", "s3://a@c");
- initializationTest("s3://a@c/", "s3://a@c");
- initializationTest("s3://a@c/path", "s3://a@c");
- initializationTest("s3://c", "s3://c");
- initializationTest("s3://c/", "s3://c");
- initializationTest("s3://c/path", "s3://c");
- }
-
- private void initializationTest(String initializationUri, String expectedUri)
- throws IOException {
-
- S3FileSystem fs = new S3FileSystem(new InMemoryFileSystemStore());
- fs.initialize(URI.create(initializationUri), new Configuration());
- assertEquals(URI.create(expectedUri), fs.getUri());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestS3InMemoryFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestS3InMemoryFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestS3InMemoryFileSystem.java
deleted file mode 100644
index fbdcd68..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestS3InMemoryFileSystem.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URI;
-import junit.framework.TestCase;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-
-public class TestS3InMemoryFileSystem extends TestCase {
-
- private static final String TEST_PATH = "s3://test/data.txt";
-
- private static final String TEST_DATA = "Sample data for testing.";
-
- private S3InMemoryFileSystem fs;
-
- @Override
- public void setUp() throws IOException {
- fs = new S3InMemoryFileSystem();
- fs.initialize(URI.create("s3://test/"), new Configuration());
- }
-
- public void testBasicReadWriteIO() throws IOException {
- FSDataOutputStream writeStream = fs.create(new Path(TEST_PATH));
- writeStream.write(TEST_DATA.getBytes());
- writeStream.flush();
- writeStream.close();
-
- FSDataInputStream readStream = fs.open(new Path(TEST_PATH));
- BufferedReader br = new BufferedReader(new InputStreamReader(readStream));
- String line = "";
- StringBuffer stringBuffer = new StringBuffer();
- while ((line = br.readLine()) != null) {
- stringBuffer.append(line);
- }
- br.close();
-
- assert(TEST_DATA.equals(stringBuffer.toString()));
- }
-
- @Override
- public void tearDown() throws IOException {
- fs.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
deleted file mode 100644
index ac572aa..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import static org.apache.hadoop.fs.s3native.NativeS3FileSystem.PATH_DELIMITER;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.Time;
-
-/**
- * <p>
- * A stub implementation of {@link NativeFileSystemStore} for testing
- * {@link NativeS3FileSystem} without actually connecting to S3.
- * </p>
- */
-public class InMemoryNativeFileSystemStore implements NativeFileSystemStore {
-
- private Configuration conf;
-
- private SortedMap<String, FileMetadata> metadataMap =
- new TreeMap<String, FileMetadata>();
- private SortedMap<String, byte[]> dataMap = new TreeMap<String, byte[]>();
-
- @Override
- public void initialize(URI uri, Configuration conf) throws IOException {
- this.conf = conf;
- }
-
- @Override
- public void storeEmptyFile(String key) throws IOException {
- metadataMap.put(key, new FileMetadata(key, 0, Time.now()));
- dataMap.put(key, new byte[0]);
- }
-
- @Override
- public void storeFile(String key, File file, byte[] md5Hash)
- throws IOException {
-
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- byte[] buf = new byte[8192];
- int numRead;
- BufferedInputStream in = null;
- try {
- in = new BufferedInputStream(new FileInputStream(file));
- while ((numRead = in.read(buf)) >= 0) {
- out.write(buf, 0, numRead);
- }
- } finally {
- if (in != null) {
- in.close();
- }
- }
- metadataMap.put(key,
- new FileMetadata(key, file.length(), Time.now()));
- dataMap.put(key, out.toByteArray());
- }
-
- @Override
- public InputStream retrieve(String key) throws IOException {
- return retrieve(key, 0);
- }
-
- @Override
- public InputStream retrieve(String key, long byteRangeStart)
- throws IOException {
-
- byte[] data = dataMap.get(key);
- File file = createTempFile();
- BufferedOutputStream out = null;
- try {
- out = new BufferedOutputStream(new FileOutputStream(file));
- out.write(data, (int) byteRangeStart,
- data.length - (int) byteRangeStart);
- } finally {
- if (out != null) {
- out.close();
- }
- }
- return new FileInputStream(file);
- }
-
- private File createTempFile() throws IOException {
- File dir = new File(conf.get("fs.s3.buffer.dir"));
- if (!dir.exists() && !dir.mkdirs()) {
- throw new IOException("Cannot create S3 buffer directory: " + dir);
- }
- File result = File.createTempFile("test-", ".tmp", dir);
- result.deleteOnExit();
- return result;
- }
-
- @Override
- public FileMetadata retrieveMetadata(String key) throws IOException {
- return metadataMap.get(key);
- }
-
- @Override
- public PartialListing list(String prefix, int maxListingLength)
- throws IOException {
- return list(prefix, maxListingLength, null, false);
- }
-
- @Override
- public PartialListing list(String prefix, int maxListingLength,
- String priorLastKey, boolean recursive) throws IOException {
-
- return list(prefix, recursive ? null : PATH_DELIMITER, maxListingLength, priorLastKey);
- }
-
- private PartialListing list(String prefix, String delimiter,
- int maxListingLength, String priorLastKey) throws IOException {
-
- if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) {
- prefix += PATH_DELIMITER;
- }
-
- List<FileMetadata> metadata = new ArrayList<FileMetadata>();
- SortedSet<String> commonPrefixes = new TreeSet<String>();
- for (String key : dataMap.keySet()) {
- if (key.startsWith(prefix)) {
- if (delimiter == null) {
- metadata.add(retrieveMetadata(key));
- } else {
- int delimIndex = key.indexOf(delimiter, prefix.length());
- if (delimIndex == -1) {
- metadata.add(retrieveMetadata(key));
- } else {
- String commonPrefix = key.substring(0, delimIndex);
- commonPrefixes.add(commonPrefix);
- }
- }
- }
- if (metadata.size() + commonPrefixes.size() == maxListingLength) {
- new PartialListing(key, metadata.toArray(new FileMetadata[0]),
- commonPrefixes.toArray(new String[0]));
- }
- }
- return new PartialListing(null, metadata.toArray(new FileMetadata[0]),
- commonPrefixes.toArray(new String[0]));
- }
-
- @Override
- public void delete(String key) throws IOException {
- metadataMap.remove(key);
- dataMap.remove(key);
- }
-
- @Override
- public void copy(String srcKey, String dstKey) throws IOException {
- metadataMap.put(dstKey, metadataMap.get(srcKey));
- dataMap.put(dstKey, dataMap.get(srcKey));
- }
-
- @Override
- public void purge(String prefix) throws IOException {
- Iterator<Entry<String, FileMetadata>> i =
- metadataMap.entrySet().iterator();
- while (i.hasNext()) {
- Entry<String, FileMetadata> entry = i.next();
- if (entry.getKey().startsWith(prefix)) {
- dataMap.remove(entry.getKey());
- i.remove();
- }
- }
- }
-
- @Override
- public void dump() throws IOException {
- System.out.println(metadataMap.values());
- System.out.println(dataMap.keySet());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/Jets3tNativeS3FileSystemContractTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/Jets3tNativeS3FileSystemContractTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/Jets3tNativeS3FileSystemContractTest.java
deleted file mode 100644
index 6516c83..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/Jets3tNativeS3FileSystemContractTest.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import java.io.IOException;
-
-public class Jets3tNativeS3FileSystemContractTest
- extends NativeS3FileSystemContractBaseTest {
-
- @Override
- NativeFileSystemStore getNativeFileSystemStore() throws IOException {
- return new Jets3tNativeFileSystemStore();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
deleted file mode 100644
index f6f9ae9..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystemContractBaseTest;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3native.NativeS3FileSystem.NativeS3FsInputStream;
-
-public abstract class NativeS3FileSystemContractBaseTest
- extends FileSystemContractBaseTest {
-
- private NativeFileSystemStore store;
-
- abstract NativeFileSystemStore getNativeFileSystemStore() throws IOException;
-
- @Override
- protected void setUp() throws Exception {
- Configuration conf = new Configuration();
- store = getNativeFileSystemStore();
- fs = new NativeS3FileSystem(store);
- fs.initialize(URI.create(conf.get("test.fs.s3n.name")), conf);
- }
-
- @Override
- protected void tearDown() throws Exception {
- store.purge("test");
- super.tearDown();
- }
-
- public void testCanonicalName() throws Exception {
- assertNull("s3n doesn't support security token and shouldn't have canonical name",
- fs.getCanonicalServiceName());
- }
-
- public void testListStatusForRoot() throws Exception {
- FileStatus[] paths = fs.listStatus(path("/"));
- assertEquals("Root directory is not empty; ", 0, paths.length);
-
- Path testDir = path("/test");
- assertTrue(fs.mkdirs(testDir));
-
- paths = fs.listStatus(path("/"));
- assertEquals(1, paths.length);
- assertEquals(path("/test"), paths[0].getPath());
- }
-
- public void testNoTrailingBackslashOnBucket() throws Exception {
- assertTrue(fs.getFileStatus(new Path(fs.getUri().toString())).isDirectory());
- }
-
- private void createTestFiles(String base) throws IOException {
- store.storeEmptyFile(base + "/file1");
- store.storeEmptyFile(base + "/dir/file2");
- store.storeEmptyFile(base + "/dir/file3");
- }
-
- public void testDirWithDifferentMarkersWorks() throws Exception {
-
- for (int i = 0; i < 3; i++) {
- String base = "test/hadoop" + i;
- Path path = path("/" + base);
-
- createTestFiles(base);
-
- if (i == 0 ) {
- //do nothing, we are testing correctness with no markers
- }
- else if (i == 1) {
- // test for _$folder$ marker
- store.storeEmptyFile(base + "_$folder$");
- store.storeEmptyFile(base + "/dir_$folder$");
- }
- else if (i == 2) {
- // test the end slash file marker
- store.storeEmptyFile(base + "/");
- store.storeEmptyFile(base + "/dir/");
- }
- else if (i == 3) {
- // test both markers
- store.storeEmptyFile(base + "_$folder$");
- store.storeEmptyFile(base + "/dir_$folder$");
- store.storeEmptyFile(base + "/");
- store.storeEmptyFile(base + "/dir/");
- }
-
- assertTrue(fs.getFileStatus(path).isDirectory());
- assertEquals(2, fs.listStatus(path).length);
- }
- }
-
- public void testDeleteWithNoMarker() throws Exception {
- String base = "test/hadoop";
- Path path = path("/" + base);
-
- createTestFiles(base);
-
- fs.delete(path, true);
-
- path = path("/test");
- assertTrue(fs.getFileStatus(path).isDirectory());
- assertEquals(0, fs.listStatus(path).length);
- }
-
- public void testRenameWithNoMarker() throws Exception {
- String base = "test/hadoop";
- Path dest = path("/test/hadoop2");
-
- createTestFiles(base);
-
- fs.rename(path("/" + base), dest);
-
- Path path = path("/test");
- assertTrue(fs.getFileStatus(path).isDirectory());
- assertEquals(1, fs.listStatus(path).length);
- assertTrue(fs.getFileStatus(dest).isDirectory());
- assertEquals(2, fs.listStatus(dest).length);
- }
-
- public void testEmptyFile() throws Exception {
- store.storeEmptyFile("test/hadoop/file1");
- fs.open(path("/test/hadoop/file1")).close();
- }
-
- public void testBlockSize() throws Exception {
- Path file = path("/test/hadoop/file");
- createFile(file);
- assertEquals("Default block size", fs.getDefaultBlockSize(file),
- fs.getFileStatus(file).getBlockSize());
-
- // Block size is determined at read time
- long newBlockSize = fs.getDefaultBlockSize(file) * 2;
- fs.getConf().setLong("fs.s3n.block.size", newBlockSize);
- assertEquals("Double default block size", newBlockSize,
- fs.getFileStatus(file).getBlockSize());
- }
-
- public void testRetryOnIoException() throws Exception {
- class TestInputStream extends InputStream {
- boolean shouldThrow = false;
- int throwCount = 0;
- int pos = 0;
- byte[] bytes;
-
- public TestInputStream() {
- bytes = new byte[256];
- for (int i = 0; i < 256; i++) {
- bytes[i] = (byte)i;
- }
- }
-
- @Override
- public int read() throws IOException {
- shouldThrow = !shouldThrow;
- if (shouldThrow) {
- throwCount++;
- throw new IOException();
- }
- return pos++;
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- shouldThrow = !shouldThrow;
- if (shouldThrow) {
- throwCount++;
- throw new IOException();
- }
-
- int sizeToRead = Math.min(len, 256 - pos);
- for (int i = 0; i < sizeToRead; i++) {
- b[i] = bytes[pos + i];
- }
- pos += sizeToRead;
- return sizeToRead;
- }
- }
-
- final InputStream is = new TestInputStream();
-
- class MockNativeFileSystemStore extends Jets3tNativeFileSystemStore {
- @Override
- public InputStream retrieve(String key, long byteRangeStart) throws IOException {
- return is;
- }
- }
-
- NativeS3FsInputStream stream = new NativeS3FsInputStream(new MockNativeFileSystemStore(), null, is, "");
-
- // Test reading methods.
- byte[] result = new byte[256];
- for (int i = 0; i < 128; i++) {
- result[i] = (byte)stream.read();
- }
- for (int i = 128; i < 256; i += 8) {
- byte[] temp = new byte[8];
- int read = stream.read(temp, 0, 8);
- assertEquals(8, read);
- System.arraycopy(temp, 0, result, i, 8);
- }
-
- // Assert correct
- for (int i = 0; i < 256; i++) {
- assertEquals((byte)i, result[i]);
- }
-
- // Test to make sure the throw path was exercised.
- // 144 = 128 + (128 / 8)
- assertEquals(144, ((TestInputStream)is).throwCount);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/S3NInMemoryFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/S3NInMemoryFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/S3NInMemoryFileSystem.java
deleted file mode 100644
index c0ea85b..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/S3NInMemoryFileSystem.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import org.apache.hadoop.fs.s3native.NativeS3FileSystem;
-import org.apache.hadoop.fs.s3native.InMemoryNativeFileSystemStore;
-
-/**
- * A helper implementation of {@link NativeS3FileSystem}
- * without actually connecting to S3 for unit testing.
- */
-public class S3NInMemoryFileSystem extends NativeS3FileSystem {
- public S3NInMemoryFileSystem() {
- super(new InMemoryNativeFileSystemStore());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
deleted file mode 100644
index 664d39e..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import java.io.IOException;
-
-public class TestInMemoryNativeS3FileSystemContract
- extends NativeS3FileSystemContractBaseTest {
-
- @Override
- NativeFileSystemStore getNativeFileSystemStore() throws IOException {
- return new InMemoryNativeFileSystemStore();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
deleted file mode 100644
index b1078a4..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-
-import static org.junit.Assert.*;
-import static org.junit.Assume.*;
-
-import org.junit.Before;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.security.DigestInputStream;
-import java.security.DigestOutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-
-public class TestJets3tNativeFileSystemStore {
- private Configuration conf;
- private Jets3tNativeFileSystemStore store;
- private NativeS3FileSystem fs;
-
- @Before
- public void setUp() throws Exception {
- conf = new Configuration();
- store = new Jets3tNativeFileSystemStore();
- fs = new NativeS3FileSystem(store);
- conf.setBoolean("fs.s3n.multipart.uploads.enabled", true);
- conf.setLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024);
- fs.initialize(URI.create(conf.get("test.fs.s3n.name")), conf);
- }
-
- @After
- public void tearDown() throws Exception {
- try {
- store.purge("test");
- } catch (Exception e) {}
- }
-
- @BeforeClass
- public static void checkSettings() throws Exception {
- Configuration conf = new Configuration();
- assumeNotNull(conf.get("fs.s3n.awsAccessKeyId"));
- assumeNotNull(conf.get("fs.s3n.awsSecretAccessKey"));
- assumeNotNull(conf.get("test.fs.s3n.name"));
- }
-
- protected void writeRenameReadCompare(Path path, long len)
- throws IOException, NoSuchAlgorithmException {
- // If len > fs.s3n.multipart.uploads.block.size,
- // we'll use a multipart upload copy
- MessageDigest digest = MessageDigest.getInstance("MD5");
- OutputStream out = new BufferedOutputStream(
- new DigestOutputStream(fs.create(path, false), digest));
- for (long i = 0; i < len; i++) {
- out.write('Q');
- }
- out.flush();
- out.close();
-
- assertTrue("Exists", fs.exists(path));
-
- // Depending on if this file is over 5 GB or not,
- // rename will cause a multipart upload copy
- Path copyPath = path.suffix(".copy");
- fs.rename(path, copyPath);
-
- assertTrue("Copy exists", fs.exists(copyPath));
-
- // Download file from S3 and compare the digest against the original
- MessageDigest digest2 = MessageDigest.getInstance("MD5");
- InputStream in = new BufferedInputStream(
- new DigestInputStream(fs.open(copyPath), digest2));
- long copyLen = 0;
- while (in.read() != -1) {copyLen++;}
- in.close();
-
- assertEquals("Copy length matches original", len, copyLen);
- assertArrayEquals("Digests match", digest.digest(), digest2.digest());
- }
-
- @Test
- public void testSmallUpload() throws IOException, NoSuchAlgorithmException {
- // Regular upload, regular copy
- writeRenameReadCompare(new Path("/test/small"), 16384);
- }
-
- @Test
- public void testMediumUpload() throws IOException, NoSuchAlgorithmException {
- // Multipart upload, regular copy
- writeRenameReadCompare(new Path("/test/medium"), 33554432); // 100 MB
- }
-
- @Test
- public void testExtraLargeUpload()
- throws IOException, NoSuchAlgorithmException {
- // Multipart upload, multipart copy
- writeRenameReadCompare(new Path("/test/xlarge"), 5368709121L); // 5GB+1byte
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/TestS3NInMemoryFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/TestS3NInMemoryFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/TestS3NInMemoryFileSystem.java
deleted file mode 100644
index b457df2..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/TestS3NInMemoryFileSystem.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URI;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-
-public class TestS3NInMemoryFileSystem extends TestCase {
-
- private static final String TEST_PATH = "s3n://test/data.txt";
-
- private static final String TEST_DATA = "Sample data for testing.";
-
- private S3NInMemoryFileSystem fs;
-
- @Override
- public void setUp() throws IOException {
- fs = new S3NInMemoryFileSystem();
- fs.initialize(URI.create("s3n://test/"), new Configuration());
- }
-
- public void testBasicReadWriteIO() throws IOException {
- FSDataOutputStream writeData = fs.create(new Path(TEST_PATH));
- writeData.write(TEST_DATA.getBytes());
- writeData.flush();
- writeData.close();
-
- FSDataInputStream readData = fs.open(new Path(TEST_PATH));
- BufferedReader br = new BufferedReader(new InputStreamReader(readData));
- String line = "";
- StringBuffer stringBuffer = new StringBuffer();
- while ((line = br.readLine()) != null) {
- stringBuffer.append(line);
- }
- br.close();
-
- assert(TEST_DATA.equals(stringBuffer.toString()));
- }
-
- @Override
- public void tearDown() throws IOException {
- fs.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/resources/contract/s3n.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/contract/s3n.xml b/hadoop-common-project/hadoop-common/src/test/resources/contract/s3n.xml
deleted file mode 100644
index ab46178..0000000
--- a/hadoop-common-project/hadoop-common/src/test/resources/contract/s3n.xml
+++ /dev/null
@@ -1,95 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration>
- <!--
- S3N is a blobstore, with very different behavior than a
- classic filesystem.
- -->
-
- <property>
- <name>fs.contract.test.root-tests-enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.test.random-seek-count</name>
- <value>10</value>
- </property>
-
- <property>
- <name>fs.contract.is-blobstore</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.is-case-sensitive</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.rename-returns-false-if-source-missing</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-append</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-atomic-directory-delete</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-atomic-rename</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-block-locality</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-concat</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-seek</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.rejects-seek-past-eof</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-strict-exceptions</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-unix-permissions</name>
- <value>false</value>
- </property>
-
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 76448cf..ad8422f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -328,6 +328,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>11.0.2</version>
@@ -576,6 +582,12 @@
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.7.2</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.mina</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
index 74e4923..204e6ab 100644
--- a/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
+++ b/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
@@ -15,5 +15,361 @@
limitations under the License.
-->
<FindBugsFilter>
+ <Match>
+ <Package name="org.apache.hadoop.security.proto" />
+ </Match>
+ <Match>
+ <Package name="org.apache.hadoop.tools.proto" />
+ </Match>
+ <Match>
+ <Bug pattern="EI_EXPOSE_REP" />
+ </Match>
+ <Match>
+ <Bug pattern="EI_EXPOSE_REP2" />
+ </Match>
+ <Match>
+ <Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
+ </Match>
+ <Match>
+ <Class name="~.*_jsp" />
+ <Bug pattern="DLS_DEAD_LOCAL_STORE" />
+ </Match>
+ <Match>
+ <Field name="_jspx_dependants" />
+ <Bug pattern="UWF_UNWRITTEN_FIELD" />
+ </Match>
+ <!--
+ Inconsistent synchronization for Client.Connection.out is
+ is intentional to make a connection to be closed instantly.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.ipc.Client$Connection" />
+ <Field name="out" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
+ <!--
+ Further SaslException should be ignored during cleanup and
+ original exception should be re-thrown.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.security.SaslRpcClient" />
+ <Bug pattern="DE_MIGHT_IGNORE" />
+ </Match>
+ <!--
+ Ignore Cross Scripting Vulnerabilities
+ -->
+ <Match>
+ <Package name="~org.apache.hadoop.mapred.*" />
+ <Bug code="XSS" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.mapred.taskdetails_jsp" />
+ <Bug code="HRS" />
+ </Match>
+ <!--
+ Ignore warnings where child class has the same name as
+ super class. Classes based on Old API shadow names from
+ new API. Should go off after HADOOP-1.0
+ -->
+ <Match>
+ <Class name="~org.apache.hadoop.mapred.*" />
+ <Bug pattern="NM_SAME_SIMPLE_NAME_AS_SUPERCLASS" />
+ </Match>
+ <!--
+ Ignore warnings for usage of System.exit. This is
+ required and have been well thought out
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.Child$2" />
+ <Method name="run" />
+ <Bug pattern="DM_EXIT" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.mapred.JobTracker" />
+ <Method name="addHostToNodeMapping" />
+ <Bug pattern="DM_EXIT" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.mapred.Task" />
+ <Or>
+ <Method name="done" />
+ <Method name="commit" />
+ <Method name="statusUpdate" />
+ </Or>
+ <Bug pattern="DM_EXIT" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.mapred.Task$TaskReporter" />
+ <Method name="run" />
+ <Bug pattern="DM_EXIT" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.util.ProgramDriver" />
+ <Method name="driver" />
+ <Bug pattern="DM_EXIT" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.util.RunJar" />
+ <Method name="run" />
+ <Bug pattern="DM_EXIT" />
+ </Match>
+ <!--
+ We need to cast objects between old and new api objects
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.OutputCommitter" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
+ <!--
+ We intentionally do the get name from the inner class
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.TaskTracker$MapEventsFetcherThread" />
+ <Method name="run" />
+ <Bug pattern="IA_AMBIGUOUS_INVOCATION_OF_INHERITED_OR_OUTER_METHOD" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.mapred.FileOutputCommitter" />
+ <Bug pattern="NM_WRONG_PACKAGE_INTENTIONAL" />
+ </Match>
+ <!--
+ Ignoring this warning as resolving this would need a non-trivial change in code
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorBaseDescriptor" />
+ <Method name="configure" />
+ <Field name="maxNumItems" />
+ <Bug pattern="ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD" />
+ </Match>
+ <!--
+ Comes from org.apache.jasper.runtime.ResourceInjector. Cannot do much.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.jobqueue_005fdetails_jsp" />
+ <Field name="_jspx_resourceInjector" />
+ <Bug pattern="SE_BAD_FIELD" />
+ </Match>
+ <!--
+ Storing textInputFormat and then passing it as a parameter. Safe to ignore.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJob" />
+ <Method name="createValueAggregatorJob" />
+ <Bug pattern="DLS_DEAD_STORE_OF_CLASS_LITERAL" />
+ </Match>
+ <!--
+ Can remove this after the upgrade to findbugs1.3.8
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.lib.db.DBInputFormat" />
+ <Method name="getSplits" />
+ <Bug pattern="DLS_DEAD_LOCAL_STORE" />
+ </Match>
+ <!--
+ This is a spurious warning. Just ignore
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.MapTask$MapOutputBuffer" />
+ <Field name="kvindex" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
- </FindBugsFilter>
+ <!--
+ core changes
+ -->
+ <Match>
+ <Class name="~org.apache.hadoop.*" />
+ <Bug code="MS" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.fs.FileSystem" />
+ <Method name="checkPath" />
+ <Bug pattern="ES_COMPARING_STRINGS_WITH_EQ" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.io.Closeable" />
+ <Bug pattern="NM_SAME_SIMPLE_NAME_AS_INTERFACE" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.security.AccessControlException" />
+ <Bug pattern="NM_SAME_SIMPLE_NAME_AS_SUPERCLASS" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.util.ProcfsBasedProcessTree" />
+ <Bug pattern="DMI_HARDCODED_ABSOLUTE_FILENAME" />
+ </Match>
+
+ <!--
+ Streaming, Examples
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.streaming.StreamUtil$TaskId" />
+ <Bug pattern="URF_UNREAD_FIELD" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.examples.DBCountPageView" />
+ <Method name="verify" />
+ <Bug pattern="OBL_UNSATISFIED_OBLIGATION" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.examples.ContextFactory" />
+ <Method name="setAttributes" />
+ <Bug pattern="OBL_UNSATISFIED_OBLIGATION" />
+ </Match>
+
+ <!--
+ TFile
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.io.file.tfile.Chunk$ChunkDecoder" />
+ <Method name="close" />
+ <Bug pattern="SR_NOT_CHECKED" />
+ </Match>
+ <!--
+ The purpose of skip() is to drain remaining bytes of the chunk-encoded
+ stream (one chunk at a time). The termination condition is checked by
+ checkEOF().
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.io.file.tfile.Utils" />
+ <Method name="writeVLong" />
+ <Bug pattern="SF_SWITCH_FALLTHROUGH" />
+ </Match>
+ <!--
+ The switch condition fall through is intentional and for performance
+ purposes.
+ -->
+
+ <Match>
+ <Class name="org.apache.hadoop.log.EventCounter"/>
+ <!-- backward compatibility -->
+ <Bug pattern="NM_SAME_SIMPLE_NAME_AS_SUPERCLASS"/>
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.metrics.jvm.EventCounter"/>
+ <!-- backward compatibility -->
+ <Bug pattern="NM_SAME_SIMPLE_NAME_AS_SUPERCLASS"/>
+ </Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.ipc\.protobuf\.ProtobufRpcEngineProtos.*"/>
+ </Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.ipc\.protobuf\.ProtocolInfoProtos.*"/>
+ </Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.ipc\.protobuf\.IpcConnectionContextProtos.*"/>
+ </Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.ipc\.protobuf\.RpcHeaderProtos.*"/>
+ </Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.ha\.proto\.HAServiceProtocolProtos.*"/>
+ </Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.ha\.proto\.ZKFCProtocolProtos.*"/>
+ </Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.security\.proto\.SecurityProtos.*"/>
+ </Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.ipc\.protobuf\.TestProtos.*"/>
+ </Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.ipc\.proto\.RefreshCallQueueProtocolProtos.*"/>
+ </Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.ipc\.proto\.GenericRefreshProtocolProtos.*"/>
+ </Match>
+
+ <!--
+ Manually checked, misses child thread manually syncing on parent's intrinsic lock.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.metrics2.lib.MutableQuantiles" />
+ <Field name="previousSnapshot" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
+ <!--
+ The method uses a generic type T that extends two other types
+ T1 and T2. Findbugs complains of a cast from T1 to T2.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.fs.DelegationTokenRenewer" />
+ <Method name="removeRenewAction" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
+
+ <!-- Inconsistent synchronization flagged by findbugs is not valid. -->
+ <Match>
+ <Class name="org.apache.hadoop.ipc.Client$Connection" />
+ <Field name="in" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
+ <!--
+ The switch condition for INITIATE is expected to fallthru to RESPONSE
+ to process initial sasl response token included in the INITIATE
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.ipc.Server$Connection" />
+ <Method name="processSaslMessage" />
+ <Bug pattern="SF_SWITCH_FALLTHROUGH" />
+ </Match>
+
+ <!-- Synchronization performed on util.concurrent instance. -->
+ <Match>
+ <Class name="org.apache.hadoop.service.AbstractService" />
+ <Method name="stop" />
+ <Bug code="JLM" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.service.AbstractService" />
+ <Method name="waitForServiceToStop" />
+ <Bug code="JLM" />
+ </Match>
+
+ <!--
+ OpenStack Swift FS module -closes streams in a different method
+ from where they are opened.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
+ <Method name="uploadFileAttempt"/>
+ <Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
+ <Method name="uploadFilePartAttempt"/>
+ <Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
+ </Match>
+
+ <!-- code from maven source, null value is checked at callee side. -->
+ <Match>
+ <Class name="org.apache.hadoop.util.ComparableVersion$ListItem" />
+ <Method name="compareTo" />
+ <Bug code="NP" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.util.HttpExceptionUtils"/>
+ <Method name="validateResponse"/>
+ <Bug pattern="REC_CATCH_EXCEPTION"/>
+ </Match>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/Block.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/Block.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/Block.java
new file mode 100644
index 0000000..6926f17
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/Block.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Holds metadata about a block of data being stored in a {@link FileSystemStore}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class Block {
+ private long id;
+
+ private long length;
+
+ public Block(long id, long length) {
+ this.id = id;
+ this.length = length;
+ }
+
+ public long getId() {
+ return id;
+ }
+
+ public long getLength() {
+ return length;
+ }
+
+ @Override
+ public String toString() {
+ return "Block[" + id + ", " + length + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/FileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/FileSystemStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/FileSystemStore.java
new file mode 100644
index 0000000..07e456b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/FileSystemStore.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * A facility for storing and retrieving {@link INode}s and {@link Block}s.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface FileSystemStore {
+
+ void initialize(URI uri, Configuration conf) throws IOException;
+ String getVersion() throws IOException;
+
+ void storeINode(Path path, INode inode) throws IOException;
+ void storeBlock(Block block, File file) throws IOException;
+
+ boolean inodeExists(Path path) throws IOException;
+ boolean blockExists(long blockId) throws IOException;
+
+ INode retrieveINode(Path path) throws IOException;
+ File retrieveBlock(Block block, long byteRangeStart) throws IOException;
+
+ void deleteINode(Path path) throws IOException;
+ void deleteBlock(Block block) throws IOException;
+
+ Set<Path> listSubPaths(Path path) throws IOException;
+ Set<Path> listDeepSubPaths(Path path) throws IOException;
+
+ /**
+ * Delete everything. Used for testing.
+ * @throws IOException
+ */
+ void purge() throws IOException;
+
+ /**
+ * Diagnostic method to dump all INodes to the console.
+ * @throws IOException
+ */
+ void dump() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/INode.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/INode.java
new file mode 100644
index 0000000..5d08b77
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/INode.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.IOUtils;
+
+/**
+ * Holds file metadata including type (regular file, or directory),
+ * and the list of blocks that are pointers to the data.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class INode {
+
+ enum FileType {
+ DIRECTORY, FILE
+ }
+
+ public static final FileType[] FILE_TYPES = {
+ FileType.DIRECTORY,
+ FileType.FILE
+ };
+
+ public static final INode DIRECTORY_INODE = new INode(FileType.DIRECTORY, null);
+
+ private FileType fileType;
+ private Block[] blocks;
+
+ public INode(FileType fileType, Block[] blocks) {
+ this.fileType = fileType;
+ if (isDirectory() && blocks != null) {
+ throw new IllegalArgumentException("A directory cannot contain blocks.");
+ }
+ this.blocks = blocks;
+ }
+
+ public Block[] getBlocks() {
+ return blocks;
+ }
+
+ public FileType getFileType() {
+ return fileType;
+ }
+
+ public boolean isDirectory() {
+ return fileType == FileType.DIRECTORY;
+ }
+
+ public boolean isFile() {
+ return fileType == FileType.FILE;
+ }
+
+ public long getSerializedLength() {
+ return 1L + (blocks == null ? 0 : 4 + blocks.length * 16);
+ }
+
+
+ public InputStream serialize() throws IOException {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(bytes);
+ try {
+ out.writeByte(fileType.ordinal());
+ if (isFile()) {
+ out.writeInt(blocks.length);
+ for (int i = 0; i < blocks.length; i++) {
+ out.writeLong(blocks[i].getId());
+ out.writeLong(blocks[i].getLength());
+ }
+ }
+ out.close();
+ out = null;
+ } finally {
+ IOUtils.closeStream(out);
+ }
+ return new ByteArrayInputStream(bytes.toByteArray());
+ }
+
+ public static INode deserialize(InputStream in) throws IOException {
+ if (in == null) {
+ return null;
+ }
+ DataInputStream dataIn = new DataInputStream(in);
+ FileType fileType = INode.FILE_TYPES[dataIn.readByte()];
+ switch (fileType) {
+ case DIRECTORY:
+ in.close();
+ return INode.DIRECTORY_INODE;
+ case FILE:
+ int numBlocks = dataIn.readInt();
+ Block[] blocks = new Block[numBlocks];
+ for (int i = 0; i < numBlocks; i++) {
+ long id = dataIn.readLong();
+ long length = dataIn.readLong();
+ blocks[i] = new Block(id, length);
+ }
+ in.close();
+ return new INode(fileType, blocks);
+ default:
+ throw new IllegalArgumentException("Cannot deserialize inode.");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
new file mode 100644
index 0000000..241ec0f
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
@@ -0,0 +1,445 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3.INode.FileType;
+import org.jets3t.service.S3Service;
+import org.jets3t.service.S3ServiceException;
+import org.jets3t.service.ServiceException;
+import org.jets3t.service.impl.rest.httpclient.RestS3Service;
+import org.jets3t.service.model.S3Bucket;
+import org.jets3t.service.model.S3Object;
+import org.jets3t.service.security.AWSCredentials;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class Jets3tFileSystemStore implements FileSystemStore {
+
+ private static final String FILE_SYSTEM_NAME = "fs";
+ private static final String FILE_SYSTEM_VALUE = "Hadoop";
+
+ private static final String FILE_SYSTEM_TYPE_NAME = "fs-type";
+ private static final String FILE_SYSTEM_TYPE_VALUE = "block";
+
+ private static final String FILE_SYSTEM_VERSION_NAME = "fs-version";
+ private static final String FILE_SYSTEM_VERSION_VALUE = "1";
+
+ private static final Map<String, Object> METADATA =
+ new HashMap<String, Object>();
+
+ static {
+ METADATA.put(FILE_SYSTEM_NAME, FILE_SYSTEM_VALUE);
+ METADATA.put(FILE_SYSTEM_TYPE_NAME, FILE_SYSTEM_TYPE_VALUE);
+ METADATA.put(FILE_SYSTEM_VERSION_NAME, FILE_SYSTEM_VERSION_VALUE);
+ }
+
+ private static final String PATH_DELIMITER = Path.SEPARATOR;
+ private static final String BLOCK_PREFIX = "block_";
+
+ private Configuration conf;
+
+ private S3Service s3Service;
+
+ private S3Bucket bucket;
+
+ private int bufferSize;
+
+ private static final Log LOG =
+ LogFactory.getLog(Jets3tFileSystemStore.class.getName());
+
+ @Override
+ public void initialize(URI uri, Configuration conf) throws IOException {
+
+ this.conf = conf;
+
+ S3Credentials s3Credentials = new S3Credentials();
+ s3Credentials.initialize(uri, conf);
+ try {
+ AWSCredentials awsCredentials =
+ new AWSCredentials(s3Credentials.getAccessKey(),
+ s3Credentials.getSecretAccessKey());
+ this.s3Service = new RestS3Service(awsCredentials);
+ } catch (S3ServiceException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ }
+ bucket = new S3Bucket(uri.getHost());
+
+ this.bufferSize = conf.getInt(
+ S3FileSystemConfigKeys.S3_STREAM_BUFFER_SIZE_KEY,
+ S3FileSystemConfigKeys.S3_STREAM_BUFFER_SIZE_DEFAULT
+ );
+ }
+
+ @Override
+ public String getVersion() throws IOException {
+ return FILE_SYSTEM_VERSION_VALUE;
+ }
+
+ private void delete(String key) throws IOException {
+ try {
+ s3Service.deleteObject(bucket, key);
+ } catch (S3ServiceException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ }
+ }
+
+ @Override
+ public void deleteINode(Path path) throws IOException {
+ delete(pathToKey(path));
+ }
+
+ @Override
+ public void deleteBlock(Block block) throws IOException {
+ delete(blockToKey(block));
+ }
+
+ @Override
+ public boolean inodeExists(Path path) throws IOException {
+ String key = pathToKey(path);
+ InputStream in = get(key, true);
+ if (in == null) {
+ if (isRoot(key)) {
+ storeINode(path, INode.DIRECTORY_INODE);
+ return true;
+ } else {
+ return false;
+ }
+ }
+ in.close();
+ return true;
+ }
+
+ @Override
+ public boolean blockExists(long blockId) throws IOException {
+ InputStream in = get(blockToKey(blockId), false);
+ if (in == null) {
+ return false;
+ }
+ in.close();
+ return true;
+ }
+
+ private InputStream get(String key, boolean checkMetadata)
+ throws IOException {
+
+ try {
+ S3Object object = s3Service.getObject(bucket.getName(), key);
+ if (checkMetadata) {
+ checkMetadata(object);
+ }
+ return object.getDataInputStream();
+ } catch (S3ServiceException e) {
+ if ("NoSuchKey".equals(e.getS3ErrorCode())) {
+ return null;
+ }
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ } catch (ServiceException e) {
+ handleServiceException(e);
+ return null;
+ }
+ }
+
+ private InputStream get(String key, long byteRangeStart) throws IOException {
+ try {
+ S3Object object = s3Service.getObject(bucket, key, null, null, null,
+ null, byteRangeStart, null);
+ return object.getDataInputStream();
+ } catch (S3ServiceException e) {
+ if ("NoSuchKey".equals(e.getS3ErrorCode())) {
+ return null;
+ }
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ } catch (ServiceException e) {
+ handleServiceException(e);
+ return null;
+ }
+ }
+
+ private void checkMetadata(S3Object object) throws S3FileSystemException,
+ S3ServiceException {
+
+ String name = (String) object.getMetadata(FILE_SYSTEM_NAME);
+ if (!FILE_SYSTEM_VALUE.equals(name)) {
+ throw new S3FileSystemException("Not a Hadoop S3 file.");
+ }
+ String type = (String) object.getMetadata(FILE_SYSTEM_TYPE_NAME);
+ if (!FILE_SYSTEM_TYPE_VALUE.equals(type)) {
+ throw new S3FileSystemException("Not a block file.");
+ }
+ String dataVersion = (String) object.getMetadata(FILE_SYSTEM_VERSION_NAME);
+ if (!FILE_SYSTEM_VERSION_VALUE.equals(dataVersion)) {
+ throw new VersionMismatchException(FILE_SYSTEM_VERSION_VALUE,
+ dataVersion);
+ }
+ }
+
+ @Override
+ public INode retrieveINode(Path path) throws IOException {
+ String key = pathToKey(path);
+ InputStream in = get(key, true);
+ if (in == null && isRoot(key)) {
+ storeINode(path, INode.DIRECTORY_INODE);
+ return INode.DIRECTORY_INODE;
+ }
+ return INode.deserialize(in);
+ }
+
+ @Override
+ public File retrieveBlock(Block block, long byteRangeStart)
+ throws IOException {
+ File fileBlock = null;
+ InputStream in = null;
+ OutputStream out = null;
+ try {
+ fileBlock = newBackupFile();
+ in = get(blockToKey(block), byteRangeStart);
+ out = new BufferedOutputStream(new FileOutputStream(fileBlock));
+ byte[] buf = new byte[bufferSize];
+ int numRead;
+ while ((numRead = in.read(buf)) >= 0) {
+ out.write(buf, 0, numRead);
+ }
+ return fileBlock;
+ } catch (IOException e) {
+ // close output stream to file then delete file
+ closeQuietly(out);
+ out = null; // to prevent a second close
+ if (fileBlock != null) {
+ boolean b = fileBlock.delete();
+ if (!b) {
+ LOG.warn("Ignoring failed delete");
+ }
+ }
+ throw e;
+ } finally {
+ closeQuietly(out);
+ closeQuietly(in);
+ }
+ }
+
+ private File newBackupFile() throws IOException {
+ File dir = new File(conf.get("fs.s3.buffer.dir"));
+ if (!dir.exists() && !dir.mkdirs()) {
+ throw new IOException("Cannot create S3 buffer directory: " + dir);
+ }
+ File result = File.createTempFile("input-", ".tmp", dir);
+ result.deleteOnExit();
+ return result;
+ }
+
+ @Override
+ public Set<Path> listSubPaths(Path path) throws IOException {
+ try {
+ String prefix = pathToKey(path);
+ if (!prefix.endsWith(PATH_DELIMITER)) {
+ prefix += PATH_DELIMITER;
+ }
+ S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, PATH_DELIMITER);
+ Set<Path> prefixes = new TreeSet<Path>();
+ for (int i = 0; i < objects.length; i++) {
+ prefixes.add(keyToPath(objects[i].getKey()));
+ }
+ prefixes.remove(path);
+ return prefixes;
+ } catch (S3ServiceException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ }
+ }
+
+ @Override
+ public Set<Path> listDeepSubPaths(Path path) throws IOException {
+ try {
+ String prefix = pathToKey(path);
+ if (!prefix.endsWith(PATH_DELIMITER)) {
+ prefix += PATH_DELIMITER;
+ }
+ S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null);
+ Set<Path> prefixes = new TreeSet<Path>();
+ for (int i = 0; i < objects.length; i++) {
+ prefixes.add(keyToPath(objects[i].getKey()));
+ }
+ prefixes.remove(path);
+ return prefixes;
+ } catch (S3ServiceException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ }
+ }
+
+ private void put(String key, InputStream in, long length, boolean storeMetadata)
+ throws IOException {
+
+ try {
+ S3Object object = new S3Object(key);
+ object.setDataInputStream(in);
+ object.setContentType("binary/octet-stream");
+ object.setContentLength(length);
+ if (storeMetadata) {
+ object.addAllMetadata(METADATA);
+ }
+ s3Service.putObject(bucket, object);
+ } catch (S3ServiceException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ }
+ }
+
+ @Override
+ public void storeINode(Path path, INode inode) throws IOException {
+ put(pathToKey(path), inode.serialize(), inode.getSerializedLength(), true);
+ }
+
+ @Override
+ public void storeBlock(Block block, File file) throws IOException {
+ BufferedInputStream in = null;
+ try {
+ in = new BufferedInputStream(new FileInputStream(file));
+ put(blockToKey(block), in, block.getLength(), false);
+ } finally {
+ closeQuietly(in);
+ }
+ }
+
+ private void closeQuietly(Closeable closeable) {
+ if (closeable != null) {
+ try {
+ closeable.close();
+ } catch (IOException e) {
+ // ignore
+ }
+ }
+ }
+
+ private String pathToKey(Path path) {
+ if (!path.isAbsolute()) {
+ throw new IllegalArgumentException("Path must be absolute: " + path);
+ }
+ return path.toUri().getPath();
+ }
+
+ private Path keyToPath(String key) {
+ return new Path(key);
+ }
+
+ private String blockToKey(long blockId) {
+ return BLOCK_PREFIX + blockId;
+ }
+
+ private String blockToKey(Block block) {
+ return blockToKey(block.getId());
+ }
+
+ private boolean isRoot(String key) {
+ return key.isEmpty() || key.equals("/");
+ }
+
+ @Override
+ public void purge() throws IOException {
+ try {
+ S3Object[] objects = s3Service.listObjects(bucket.getName());
+ for (int i = 0; i < objects.length; i++) {
+ s3Service.deleteObject(bucket, objects[i].getKey());
+ }
+ } catch (S3ServiceException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ }
+ }
+
+ @Override
+ public void dump() throws IOException {
+ StringBuilder sb = new StringBuilder("S3 Filesystem, ");
+ sb.append(bucket.getName()).append("\n");
+ try {
+ S3Object[] objects = s3Service.listObjects(bucket.getName(), PATH_DELIMITER, null);
+ for (int i = 0; i < objects.length; i++) {
+ Path path = keyToPath(objects[i].getKey());
+ sb.append(path).append("\n");
+ INode m = retrieveINode(path);
+ sb.append("\t").append(m.getFileType()).append("\n");
+ if (m.getFileType() == FileType.DIRECTORY) {
+ continue;
+ }
+ for (int j = 0; j < m.getBlocks().length; j++) {
+ sb.append("\t").append(m.getBlocks()[j]).append("\n");
+ }
+ }
+ } catch (S3ServiceException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ }
+ System.out.println(sb);
+ }
+
+ private void handleServiceException(ServiceException e) throws IOException {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ else {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Got ServiceException with Error code: " + e.getErrorCode() + ";and Error message: " + e.getErrorMessage());
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java
new file mode 100644
index 0000000..429c272
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java
@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.jets3t.service.S3Service;
+import org.jets3t.service.S3ServiceException;
+import org.jets3t.service.ServiceException;
+import org.jets3t.service.impl.rest.httpclient.RestS3Service;
+import org.jets3t.service.model.S3Bucket;
+import org.jets3t.service.model.S3Object;
+import org.jets3t.service.security.AWSCredentials;
+
+/**
+ * <p>
+ * This class is a tool for migrating data from an older to a newer version
+ * of an S3 filesystem.
+ * </p>
+ * <p>
+ * All files in the filesystem are migrated by re-writing the block metadata
+ * - no datafiles are touched.
+ * </p>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class MigrationTool extends Configured implements Tool {
+
+ private S3Service s3Service;
+ private S3Bucket bucket;
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new MigrationTool(), args);
+ System.exit(res);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+ if (args.length == 0) {
+ System.err.println("Usage: MigrationTool <S3 file system URI>");
+ System.err.println("\t<S3 file system URI>\tfilesystem to migrate");
+ ToolRunner.printGenericCommandUsage(System.err);
+ return -1;
+ }
+
+ URI uri = URI.create(args[0]);
+
+ initialize(uri);
+
+ FileSystemStore newStore = new Jets3tFileSystemStore();
+ newStore.initialize(uri, getConf());
+
+ if (get("%2F") != null) {
+ System.err.println("Current version number is [unversioned].");
+ System.err.println("Target version number is " +
+ newStore.getVersion() + ".");
+ Store oldStore = new UnversionedStore();
+ migrate(oldStore, newStore);
+ return 0;
+ } else {
+ S3Object root = get("/");
+ if (root != null) {
+ String version = (String) root.getMetadata("fs-version");
+ if (version == null) {
+ System.err.println("Can't detect version - exiting.");
+ } else {
+ String newVersion = newStore.getVersion();
+ System.err.println("Current version number is " + version + ".");
+ System.err.println("Target version number is " + newVersion + ".");
+ if (version.equals(newStore.getVersion())) {
+ System.err.println("No migration required.");
+ return 0;
+ }
+ // use version number to create Store
+ //Store oldStore = ...
+ //migrate(oldStore, newStore);
+ System.err.println("Not currently implemented.");
+ return 0;
+ }
+ }
+ System.err.println("Can't detect version - exiting.");
+ return 0;
+ }
+
+ }
+
+ public void initialize(URI uri) throws IOException {
+
+
+
+ try {
+ String accessKey = null;
+ String secretAccessKey = null;
+ String userInfo = uri.getUserInfo();
+ if (userInfo != null) {
+ int index = userInfo.indexOf(':');
+ if (index != -1) {
+ accessKey = userInfo.substring(0, index);
+ secretAccessKey = userInfo.substring(index + 1);
+ } else {
+ accessKey = userInfo;
+ }
+ }
+ if (accessKey == null) {
+ accessKey = getConf().get("fs.s3.awsAccessKeyId");
+ }
+ if (secretAccessKey == null) {
+ secretAccessKey = getConf().get("fs.s3.awsSecretAccessKey");
+ }
+ if (accessKey == null && secretAccessKey == null) {
+ throw new IllegalArgumentException("AWS " +
+ "Access Key ID and Secret Access Key " +
+ "must be specified as the username " +
+ "or password (respectively) of a s3 URL, " +
+ "or by setting the " +
+ "fs.s3.awsAccessKeyId or " +
+ "fs.s3.awsSecretAccessKey properties (respectively).");
+ } else if (accessKey == null) {
+ throw new IllegalArgumentException("AWS " +
+ "Access Key ID must be specified " +
+ "as the username of a s3 URL, or by setting the " +
+ "fs.s3.awsAccessKeyId property.");
+ } else if (secretAccessKey == null) {
+ throw new IllegalArgumentException("AWS " +
+ "Secret Access Key must be specified " +
+ "as the password of a s3 URL, or by setting the " +
+ "fs.s3.awsSecretAccessKey property.");
+ }
+ AWSCredentials awsCredentials =
+ new AWSCredentials(accessKey, secretAccessKey);
+ this.s3Service = new RestS3Service(awsCredentials);
+ } catch (S3ServiceException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ }
+ bucket = new S3Bucket(uri.getHost());
+ }
+
+ private void migrate(Store oldStore, FileSystemStore newStore)
+ throws IOException {
+ for (Path path : oldStore.listAllPaths()) {
+ INode inode = oldStore.retrieveINode(path);
+ oldStore.deleteINode(path);
+ newStore.storeINode(path, inode);
+ }
+ }
+
+ private S3Object get(String key) {
+ try {
+ return s3Service.getObject(bucket.getName(), key);
+ } catch (S3ServiceException e) {
+ if ("NoSuchKey".equals(e.getS3ErrorCode())) {
+ return null;
+ }
+ }
+ return null;
+ }
+
+ interface Store {
+
+ Set<Path> listAllPaths() throws IOException;
+ INode retrieveINode(Path path) throws IOException;
+ void deleteINode(Path path) throws IOException;
+
+ }
+
+ class UnversionedStore implements Store {
+
+ @Override
+ public Set<Path> listAllPaths() throws IOException {
+ try {
+ String prefix = urlEncode(Path.SEPARATOR);
+ S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null);
+ Set<Path> prefixes = new TreeSet<Path>();
+ for (int i = 0; i < objects.length; i++) {
+ prefixes.add(keyToPath(objects[i].getKey()));
+ }
+ return prefixes;
+ } catch (S3ServiceException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ }
+ }
+
+ @Override
+ public void deleteINode(Path path) throws IOException {
+ delete(pathToKey(path));
+ }
+
+ private void delete(String key) throws IOException {
+ try {
+ s3Service.deleteObject(bucket, key);
+ } catch (S3ServiceException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ }
+ }
+
+ @Override
+ public INode retrieveINode(Path path) throws IOException {
+ return INode.deserialize(get(pathToKey(path)));
+ }
+
+ private InputStream get(String key) throws IOException {
+ try {
+ S3Object object = s3Service.getObject(bucket.getName(), key);
+ return object.getDataInputStream();
+ } catch (S3ServiceException e) {
+ if ("NoSuchKey".equals(e.getS3ErrorCode())) {
+ return null;
+ }
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new S3Exception(e);
+ } catch (ServiceException e) {
+ return null;
+ }
+ }
+
+ private String pathToKey(Path path) {
+ if (!path.isAbsolute()) {
+ throw new IllegalArgumentException("Path must be absolute: " + path);
+ }
+ return urlEncode(path.toUri().getPath());
+ }
+
+ private Path keyToPath(String key) {
+ return new Path(urlDecode(key));
+ }
+
+ private String urlEncode(String s) {
+ try {
+ return URLEncoder.encode(s, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ // Should never happen since every implementation of the Java Platform
+ // is required to support UTF-8.
+ // See http://java.sun.com/j2se/1.5.0/docs/api/java/nio/charset/Charset.html
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private String urlDecode(String s) {
+ try {
+ return URLDecoder.decode(s, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ // Should never happen since every implementation of the Java Platform
+ // is required to support UTF-8.
+ // See http://java.sun.com/j2se/1.5.0/docs/api/java/nio/charset/Charset.html
+ throw new IllegalStateException(e);
+ }
+ }
+
+ }
+
+}
[16/20] git commit: HADOOP-11074. Move s3-related FS connector code
to hadoop-aws. (David S. Wang via Colin Patrick McCabe)
Posted by sz...@apache.org.
HADOOP-11074. Move s3-related FS connector code to hadoop-aws. (David S. Wang via Colin Patrick McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ec7fcd9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ec7fcd9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ec7fcd9
Branch: refs/heads/HDFS-6584
Commit: 5ec7fcd9dd6bb86858c6e2583321bb9a615bd392
Parents: 7f80e14
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Sep 10 16:14:08 2014 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Wed Sep 10 16:14:53 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/s3/Block.java | 52 --
.../apache/hadoop/fs/s3/FileSystemStore.java | 67 --
.../java/org/apache/hadoop/fs/s3/INode.java | 128 ---
.../hadoop/fs/s3/Jets3tFileSystemStore.java | 445 -----------
.../org/apache/hadoop/fs/s3/MigrationTool.java | 291 -------
.../org/apache/hadoop/fs/s3/S3Credentials.java | 103 ---
.../org/apache/hadoop/fs/s3/S3Exception.java | 39 -
.../org/apache/hadoop/fs/s3/S3FileSystem.java | 486 -----------
.../hadoop/fs/s3/S3FileSystemConfigKeys.java | 47 --
.../hadoop/fs/s3/S3FileSystemException.java | 36 -
.../org/apache/hadoop/fs/s3/S3InputStream.java | 215 -----
.../org/apache/hadoop/fs/s3/S3OutputStream.java | 235 ------
.../hadoop/fs/s3/VersionMismatchException.java | 37 -
.../java/org/apache/hadoop/fs/s3/package.html | 55 --
.../apache/hadoop/fs/s3native/FileMetadata.java | 59 --
.../s3native/Jets3tNativeFileSystemStore.java | 483 -----------
.../fs/s3native/NativeFileSystemStore.java | 67 --
.../hadoop/fs/s3native/NativeS3FileSystem.java | 796 -------------------
.../hadoop/fs/s3native/PartialListing.java | 64 --
.../s3native/S3NativeFileSystemConfigKeys.java | 47 --
.../org/apache/hadoop/fs/s3native/package.html | 32 -
.../services/org.apache.hadoop.fs.FileSystem | 2 -
.../fs/contract/s3n/NativeS3Contract.java | 43 -
.../fs/contract/s3n/TestS3NContractCreate.java | 38 -
.../fs/contract/s3n/TestS3NContractDelete.java | 31 -
.../fs/contract/s3n/TestS3NContractMkdir.java | 34 -
.../fs/contract/s3n/TestS3NContractOpen.java | 31 -
.../fs/contract/s3n/TestS3NContractRename.java | 32 -
.../fs/contract/s3n/TestS3NContractRootDir.java | 35 -
.../fs/contract/s3n/TestS3NContractSeek.java | 31 -
.../hadoop/fs/s3/InMemoryFileSystemStore.java | 200 -----
.../fs/s3/Jets3tS3FileSystemContractTest.java | 31 -
.../fs/s3/S3FileSystemContractBaseTest.java | 63 --
.../hadoop/fs/s3/S3InMemoryFileSystem.java | 32 -
.../java/org/apache/hadoop/fs/s3/TestINode.java | 60 --
.../fs/s3/TestInMemoryS3FileSystemContract.java | 31 -
.../apache/hadoop/fs/s3/TestS3Credentials.java | 36 -
.../apache/hadoop/fs/s3/TestS3FileSystem.java | 50 --
.../hadoop/fs/s3/TestS3InMemoryFileSystem.java | 67 --
.../s3native/InMemoryNativeFileSystemStore.java | 206 -----
.../Jets3tNativeS3FileSystemContractTest.java | 30 -
.../NativeS3FileSystemContractBaseTest.java | 233 ------
.../fs/s3native/S3NInMemoryFileSystem.java | 32 -
.../TestInMemoryNativeS3FileSystemContract.java | 30 -
.../TestJets3tNativeFileSystemStore.java | 126 ---
.../fs/s3native/TestS3NInMemoryFileSystem.java | 69 --
.../src/test/resources/contract/s3n.xml | 95 ---
hadoop-project/pom.xml | 12 +
.../hadoop-aws/dev-support/findbugs-exclude.xml | 358 ++++++++-
.../java/org/apache/hadoop/fs/s3/Block.java | 52 ++
.../apache/hadoop/fs/s3/FileSystemStore.java | 67 ++
.../java/org/apache/hadoop/fs/s3/INode.java | 128 +++
.../hadoop/fs/s3/Jets3tFileSystemStore.java | 445 +++++++++++
.../org/apache/hadoop/fs/s3/MigrationTool.java | 291 +++++++
.../org/apache/hadoop/fs/s3/S3Credentials.java | 103 +++
.../org/apache/hadoop/fs/s3/S3Exception.java | 39 +
.../org/apache/hadoop/fs/s3/S3FileSystem.java | 486 +++++++++++
.../hadoop/fs/s3/S3FileSystemConfigKeys.java | 47 ++
.../hadoop/fs/s3/S3FileSystemException.java | 36 +
.../org/apache/hadoop/fs/s3/S3InputStream.java | 215 +++++
.../org/apache/hadoop/fs/s3/S3OutputStream.java | 235 ++++++
.../hadoop/fs/s3/VersionMismatchException.java | 37 +
.../java/org/apache/hadoop/fs/s3/package.html | 55 ++
.../apache/hadoop/fs/s3native/FileMetadata.java | 59 ++
.../s3native/Jets3tNativeFileSystemStore.java | 483 +++++++++++
.../fs/s3native/NativeFileSystemStore.java | 67 ++
.../hadoop/fs/s3native/NativeS3FileSystem.java | 796 +++++++++++++++++++
.../hadoop/fs/s3native/PartialListing.java | 64 ++
.../s3native/S3NativeFileSystemConfigKeys.java | 47 ++
.../org/apache/hadoop/fs/s3native/package.html | 32 +
.../services/org.apache.hadoop.fs.FileSystem | 17 +
.../fs/contract/s3n/NativeS3Contract.java | 43 +
.../fs/contract/s3n/TestS3NContractCreate.java | 38 +
.../fs/contract/s3n/TestS3NContractDelete.java | 31 +
.../fs/contract/s3n/TestS3NContractMkdir.java | 34 +
.../fs/contract/s3n/TestS3NContractOpen.java | 31 +
.../fs/contract/s3n/TestS3NContractRename.java | 32 +
.../fs/contract/s3n/TestS3NContractRootDir.java | 35 +
.../fs/contract/s3n/TestS3NContractSeek.java | 31 +
.../hadoop/fs/s3/InMemoryFileSystemStore.java | 200 +++++
.../fs/s3/Jets3tS3FileSystemContractTest.java | 31 +
.../fs/s3/S3FileSystemContractBaseTest.java | 54 ++
.../hadoop/fs/s3/S3InMemoryFileSystem.java | 32 +
.../java/org/apache/hadoop/fs/s3/TestINode.java | 60 ++
.../fs/s3/TestInMemoryS3FileSystemContract.java | 31 +
.../apache/hadoop/fs/s3/TestS3Credentials.java | 36 +
.../apache/hadoop/fs/s3/TestS3FileSystem.java | 50 ++
.../hadoop/fs/s3/TestS3InMemoryFileSystem.java | 67 ++
.../s3native/InMemoryNativeFileSystemStore.java | 206 +++++
.../Jets3tNativeS3FileSystemContractTest.java | 30 +
.../NativeS3FileSystemContractBaseTest.java | 233 ++++++
.../fs/s3native/S3NInMemoryFileSystem.java | 32 +
.../TestInMemoryNativeS3FileSystemContract.java | 30 +
.../TestJets3tNativeFileSystemStore.java | 126 +++
.../fs/s3native/TestS3NInMemoryFileSystem.java | 69 ++
.../src/test/resources/contract/s3n.xml | 95 +++
hadoop-tools/hadoop-tools-dist/pom.xml | 6 +
97 files changed, 5833 insertions(+), 5453 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Block.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Block.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Block.java
deleted file mode 100644
index 6926f17..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Block.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Holds metadata about a block of data being stored in a {@link FileSystemStore}.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class Block {
- private long id;
-
- private long length;
-
- public Block(long id, long length) {
- this.id = id;
- this.length = length;
- }
-
- public long getId() {
- return id;
- }
-
- public long getLength() {
- return length;
- }
-
- @Override
- public String toString() {
- return "Block[" + id + ", " + length + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/FileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/FileSystemStore.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/FileSystemStore.java
deleted file mode 100644
index 07e456b..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/FileSystemStore.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.util.Set;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-
-/**
- * A facility for storing and retrieving {@link INode}s and {@link Block}s.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public interface FileSystemStore {
-
- void initialize(URI uri, Configuration conf) throws IOException;
- String getVersion() throws IOException;
-
- void storeINode(Path path, INode inode) throws IOException;
- void storeBlock(Block block, File file) throws IOException;
-
- boolean inodeExists(Path path) throws IOException;
- boolean blockExists(long blockId) throws IOException;
-
- INode retrieveINode(Path path) throws IOException;
- File retrieveBlock(Block block, long byteRangeStart) throws IOException;
-
- void deleteINode(Path path) throws IOException;
- void deleteBlock(Block block) throws IOException;
-
- Set<Path> listSubPaths(Path path) throws IOException;
- Set<Path> listDeepSubPaths(Path path) throws IOException;
-
- /**
- * Delete everything. Used for testing.
- * @throws IOException
- */
- void purge() throws IOException;
-
- /**
- * Diagnostic method to dump all INodes to the console.
- * @throws IOException
- */
- void dump() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/INode.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/INode.java
deleted file mode 100644
index 5d08b77..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/INode.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.IOUtils;
-
-/**
- * Holds file metadata including type (regular file, or directory),
- * and the list of blocks that are pointers to the data.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class INode {
-
- enum FileType {
- DIRECTORY, FILE
- }
-
- public static final FileType[] FILE_TYPES = {
- FileType.DIRECTORY,
- FileType.FILE
- };
-
- public static final INode DIRECTORY_INODE = new INode(FileType.DIRECTORY, null);
-
- private FileType fileType;
- private Block[] blocks;
-
- public INode(FileType fileType, Block[] blocks) {
- this.fileType = fileType;
- if (isDirectory() && blocks != null) {
- throw new IllegalArgumentException("A directory cannot contain blocks.");
- }
- this.blocks = blocks;
- }
-
- public Block[] getBlocks() {
- return blocks;
- }
-
- public FileType getFileType() {
- return fileType;
- }
-
- public boolean isDirectory() {
- return fileType == FileType.DIRECTORY;
- }
-
- public boolean isFile() {
- return fileType == FileType.FILE;
- }
-
- public long getSerializedLength() {
- return 1L + (blocks == null ? 0 : 4 + blocks.length * 16);
- }
-
-
- public InputStream serialize() throws IOException {
- ByteArrayOutputStream bytes = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(bytes);
- try {
- out.writeByte(fileType.ordinal());
- if (isFile()) {
- out.writeInt(blocks.length);
- for (int i = 0; i < blocks.length; i++) {
- out.writeLong(blocks[i].getId());
- out.writeLong(blocks[i].getLength());
- }
- }
- out.close();
- out = null;
- } finally {
- IOUtils.closeStream(out);
- }
- return new ByteArrayInputStream(bytes.toByteArray());
- }
-
- public static INode deserialize(InputStream in) throws IOException {
- if (in == null) {
- return null;
- }
- DataInputStream dataIn = new DataInputStream(in);
- FileType fileType = INode.FILE_TYPES[dataIn.readByte()];
- switch (fileType) {
- case DIRECTORY:
- in.close();
- return INode.DIRECTORY_INODE;
- case FILE:
- int numBlocks = dataIn.readInt();
- Block[] blocks = new Block[numBlocks];
- for (int i = 0; i < numBlocks; i++) {
- long id = dataIn.readLong();
- long length = dataIn.readLong();
- blocks[i] = new Block(id, length);
- }
- in.close();
- return new INode(fileType, blocks);
- default:
- throw new IllegalArgumentException("Cannot deserialize inode.");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
deleted file mode 100644
index 241ec0f..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
+++ /dev/null
@@ -1,445 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3.INode.FileType;
-import org.jets3t.service.S3Service;
-import org.jets3t.service.S3ServiceException;
-import org.jets3t.service.ServiceException;
-import org.jets3t.service.impl.rest.httpclient.RestS3Service;
-import org.jets3t.service.model.S3Bucket;
-import org.jets3t.service.model.S3Object;
-import org.jets3t.service.security.AWSCredentials;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-class Jets3tFileSystemStore implements FileSystemStore {
-
- private static final String FILE_SYSTEM_NAME = "fs";
- private static final String FILE_SYSTEM_VALUE = "Hadoop";
-
- private static final String FILE_SYSTEM_TYPE_NAME = "fs-type";
- private static final String FILE_SYSTEM_TYPE_VALUE = "block";
-
- private static final String FILE_SYSTEM_VERSION_NAME = "fs-version";
- private static final String FILE_SYSTEM_VERSION_VALUE = "1";
-
- private static final Map<String, Object> METADATA =
- new HashMap<String, Object>();
-
- static {
- METADATA.put(FILE_SYSTEM_NAME, FILE_SYSTEM_VALUE);
- METADATA.put(FILE_SYSTEM_TYPE_NAME, FILE_SYSTEM_TYPE_VALUE);
- METADATA.put(FILE_SYSTEM_VERSION_NAME, FILE_SYSTEM_VERSION_VALUE);
- }
-
- private static final String PATH_DELIMITER = Path.SEPARATOR;
- private static final String BLOCK_PREFIX = "block_";
-
- private Configuration conf;
-
- private S3Service s3Service;
-
- private S3Bucket bucket;
-
- private int bufferSize;
-
- private static final Log LOG =
- LogFactory.getLog(Jets3tFileSystemStore.class.getName());
-
- @Override
- public void initialize(URI uri, Configuration conf) throws IOException {
-
- this.conf = conf;
-
- S3Credentials s3Credentials = new S3Credentials();
- s3Credentials.initialize(uri, conf);
- try {
- AWSCredentials awsCredentials =
- new AWSCredentials(s3Credentials.getAccessKey(),
- s3Credentials.getSecretAccessKey());
- this.s3Service = new RestS3Service(awsCredentials);
- } catch (S3ServiceException e) {
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- }
- bucket = new S3Bucket(uri.getHost());
-
- this.bufferSize = conf.getInt(
- S3FileSystemConfigKeys.S3_STREAM_BUFFER_SIZE_KEY,
- S3FileSystemConfigKeys.S3_STREAM_BUFFER_SIZE_DEFAULT
- );
- }
-
- @Override
- public String getVersion() throws IOException {
- return FILE_SYSTEM_VERSION_VALUE;
- }
-
- private void delete(String key) throws IOException {
- try {
- s3Service.deleteObject(bucket, key);
- } catch (S3ServiceException e) {
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- }
- }
-
- @Override
- public void deleteINode(Path path) throws IOException {
- delete(pathToKey(path));
- }
-
- @Override
- public void deleteBlock(Block block) throws IOException {
- delete(blockToKey(block));
- }
-
- @Override
- public boolean inodeExists(Path path) throws IOException {
- String key = pathToKey(path);
- InputStream in = get(key, true);
- if (in == null) {
- if (isRoot(key)) {
- storeINode(path, INode.DIRECTORY_INODE);
- return true;
- } else {
- return false;
- }
- }
- in.close();
- return true;
- }
-
- @Override
- public boolean blockExists(long blockId) throws IOException {
- InputStream in = get(blockToKey(blockId), false);
- if (in == null) {
- return false;
- }
- in.close();
- return true;
- }
-
- private InputStream get(String key, boolean checkMetadata)
- throws IOException {
-
- try {
- S3Object object = s3Service.getObject(bucket.getName(), key);
- if (checkMetadata) {
- checkMetadata(object);
- }
- return object.getDataInputStream();
- } catch (S3ServiceException e) {
- if ("NoSuchKey".equals(e.getS3ErrorCode())) {
- return null;
- }
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- } catch (ServiceException e) {
- handleServiceException(e);
- return null;
- }
- }
-
- private InputStream get(String key, long byteRangeStart) throws IOException {
- try {
- S3Object object = s3Service.getObject(bucket, key, null, null, null,
- null, byteRangeStart, null);
- return object.getDataInputStream();
- } catch (S3ServiceException e) {
- if ("NoSuchKey".equals(e.getS3ErrorCode())) {
- return null;
- }
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- } catch (ServiceException e) {
- handleServiceException(e);
- return null;
- }
- }
-
- private void checkMetadata(S3Object object) throws S3FileSystemException,
- S3ServiceException {
-
- String name = (String) object.getMetadata(FILE_SYSTEM_NAME);
- if (!FILE_SYSTEM_VALUE.equals(name)) {
- throw new S3FileSystemException("Not a Hadoop S3 file.");
- }
- String type = (String) object.getMetadata(FILE_SYSTEM_TYPE_NAME);
- if (!FILE_SYSTEM_TYPE_VALUE.equals(type)) {
- throw new S3FileSystemException("Not a block file.");
- }
- String dataVersion = (String) object.getMetadata(FILE_SYSTEM_VERSION_NAME);
- if (!FILE_SYSTEM_VERSION_VALUE.equals(dataVersion)) {
- throw new VersionMismatchException(FILE_SYSTEM_VERSION_VALUE,
- dataVersion);
- }
- }
-
- @Override
- public INode retrieveINode(Path path) throws IOException {
- String key = pathToKey(path);
- InputStream in = get(key, true);
- if (in == null && isRoot(key)) {
- storeINode(path, INode.DIRECTORY_INODE);
- return INode.DIRECTORY_INODE;
- }
- return INode.deserialize(in);
- }
-
- @Override
- public File retrieveBlock(Block block, long byteRangeStart)
- throws IOException {
- File fileBlock = null;
- InputStream in = null;
- OutputStream out = null;
- try {
- fileBlock = newBackupFile();
- in = get(blockToKey(block), byteRangeStart);
- out = new BufferedOutputStream(new FileOutputStream(fileBlock));
- byte[] buf = new byte[bufferSize];
- int numRead;
- while ((numRead = in.read(buf)) >= 0) {
- out.write(buf, 0, numRead);
- }
- return fileBlock;
- } catch (IOException e) {
- // close output stream to file then delete file
- closeQuietly(out);
- out = null; // to prevent a second close
- if (fileBlock != null) {
- boolean b = fileBlock.delete();
- if (!b) {
- LOG.warn("Ignoring failed delete");
- }
- }
- throw e;
- } finally {
- closeQuietly(out);
- closeQuietly(in);
- }
- }
-
- private File newBackupFile() throws IOException {
- File dir = new File(conf.get("fs.s3.buffer.dir"));
- if (!dir.exists() && !dir.mkdirs()) {
- throw new IOException("Cannot create S3 buffer directory: " + dir);
- }
- File result = File.createTempFile("input-", ".tmp", dir);
- result.deleteOnExit();
- return result;
- }
-
- @Override
- public Set<Path> listSubPaths(Path path) throws IOException {
- try {
- String prefix = pathToKey(path);
- if (!prefix.endsWith(PATH_DELIMITER)) {
- prefix += PATH_DELIMITER;
- }
- S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, PATH_DELIMITER);
- Set<Path> prefixes = new TreeSet<Path>();
- for (int i = 0; i < objects.length; i++) {
- prefixes.add(keyToPath(objects[i].getKey()));
- }
- prefixes.remove(path);
- return prefixes;
- } catch (S3ServiceException e) {
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- }
- }
-
- @Override
- public Set<Path> listDeepSubPaths(Path path) throws IOException {
- try {
- String prefix = pathToKey(path);
- if (!prefix.endsWith(PATH_DELIMITER)) {
- prefix += PATH_DELIMITER;
- }
- S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null);
- Set<Path> prefixes = new TreeSet<Path>();
- for (int i = 0; i < objects.length; i++) {
- prefixes.add(keyToPath(objects[i].getKey()));
- }
- prefixes.remove(path);
- return prefixes;
- } catch (S3ServiceException e) {
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- }
- }
-
- private void put(String key, InputStream in, long length, boolean storeMetadata)
- throws IOException {
-
- try {
- S3Object object = new S3Object(key);
- object.setDataInputStream(in);
- object.setContentType("binary/octet-stream");
- object.setContentLength(length);
- if (storeMetadata) {
- object.addAllMetadata(METADATA);
- }
- s3Service.putObject(bucket, object);
- } catch (S3ServiceException e) {
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- }
- }
-
- @Override
- public void storeINode(Path path, INode inode) throws IOException {
- put(pathToKey(path), inode.serialize(), inode.getSerializedLength(), true);
- }
-
- @Override
- public void storeBlock(Block block, File file) throws IOException {
- BufferedInputStream in = null;
- try {
- in = new BufferedInputStream(new FileInputStream(file));
- put(blockToKey(block), in, block.getLength(), false);
- } finally {
- closeQuietly(in);
- }
- }
-
- private void closeQuietly(Closeable closeable) {
- if (closeable != null) {
- try {
- closeable.close();
- } catch (IOException e) {
- // ignore
- }
- }
- }
-
- private String pathToKey(Path path) {
- if (!path.isAbsolute()) {
- throw new IllegalArgumentException("Path must be absolute: " + path);
- }
- return path.toUri().getPath();
- }
-
- private Path keyToPath(String key) {
- return new Path(key);
- }
-
- private String blockToKey(long blockId) {
- return BLOCK_PREFIX + blockId;
- }
-
- private String blockToKey(Block block) {
- return blockToKey(block.getId());
- }
-
- private boolean isRoot(String key) {
- return key.isEmpty() || key.equals("/");
- }
-
- @Override
- public void purge() throws IOException {
- try {
- S3Object[] objects = s3Service.listObjects(bucket.getName());
- for (int i = 0; i < objects.length; i++) {
- s3Service.deleteObject(bucket, objects[i].getKey());
- }
- } catch (S3ServiceException e) {
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- }
- }
-
- @Override
- public void dump() throws IOException {
- StringBuilder sb = new StringBuilder("S3 Filesystem, ");
- sb.append(bucket.getName()).append("\n");
- try {
- S3Object[] objects = s3Service.listObjects(bucket.getName(), PATH_DELIMITER, null);
- for (int i = 0; i < objects.length; i++) {
- Path path = keyToPath(objects[i].getKey());
- sb.append(path).append("\n");
- INode m = retrieveINode(path);
- sb.append("\t").append(m.getFileType()).append("\n");
- if (m.getFileType() == FileType.DIRECTORY) {
- continue;
- }
- for (int j = 0; j < m.getBlocks().length; j++) {
- sb.append("\t").append(m.getBlocks()[j]).append("\n");
- }
- }
- } catch (S3ServiceException e) {
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- }
- System.out.println(sb);
- }
-
- private void handleServiceException(ServiceException e) throws IOException {
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- else {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Got ServiceException with Error code: " + e.getErrorCode() + ";and Error message: " + e.getErrorMessage());
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java
deleted file mode 100644
index 429c272..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java
+++ /dev/null
@@ -1,291 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URLDecoder;
-import java.net.URLEncoder;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.jets3t.service.S3Service;
-import org.jets3t.service.S3ServiceException;
-import org.jets3t.service.ServiceException;
-import org.jets3t.service.impl.rest.httpclient.RestS3Service;
-import org.jets3t.service.model.S3Bucket;
-import org.jets3t.service.model.S3Object;
-import org.jets3t.service.security.AWSCredentials;
-
-/**
- * <p>
- * This class is a tool for migrating data from an older to a newer version
- * of an S3 filesystem.
- * </p>
- * <p>
- * All files in the filesystem are migrated by re-writing the block metadata
- * - no datafiles are touched.
- * </p>
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class MigrationTool extends Configured implements Tool {
-
- private S3Service s3Service;
- private S3Bucket bucket;
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new MigrationTool(), args);
- System.exit(res);
- }
-
- @Override
- public int run(String[] args) throws Exception {
-
- if (args.length == 0) {
- System.err.println("Usage: MigrationTool <S3 file system URI>");
- System.err.println("\t<S3 file system URI>\tfilesystem to migrate");
- ToolRunner.printGenericCommandUsage(System.err);
- return -1;
- }
-
- URI uri = URI.create(args[0]);
-
- initialize(uri);
-
- FileSystemStore newStore = new Jets3tFileSystemStore();
- newStore.initialize(uri, getConf());
-
- if (get("%2F") != null) {
- System.err.println("Current version number is [unversioned].");
- System.err.println("Target version number is " +
- newStore.getVersion() + ".");
- Store oldStore = new UnversionedStore();
- migrate(oldStore, newStore);
- return 0;
- } else {
- S3Object root = get("/");
- if (root != null) {
- String version = (String) root.getMetadata("fs-version");
- if (version == null) {
- System.err.println("Can't detect version - exiting.");
- } else {
- String newVersion = newStore.getVersion();
- System.err.println("Current version number is " + version + ".");
- System.err.println("Target version number is " + newVersion + ".");
- if (version.equals(newStore.getVersion())) {
- System.err.println("No migration required.");
- return 0;
- }
- // use version number to create Store
- //Store oldStore = ...
- //migrate(oldStore, newStore);
- System.err.println("Not currently implemented.");
- return 0;
- }
- }
- System.err.println("Can't detect version - exiting.");
- return 0;
- }
-
- }
-
- public void initialize(URI uri) throws IOException {
-
-
-
- try {
- String accessKey = null;
- String secretAccessKey = null;
- String userInfo = uri.getUserInfo();
- if (userInfo != null) {
- int index = userInfo.indexOf(':');
- if (index != -1) {
- accessKey = userInfo.substring(0, index);
- secretAccessKey = userInfo.substring(index + 1);
- } else {
- accessKey = userInfo;
- }
- }
- if (accessKey == null) {
- accessKey = getConf().get("fs.s3.awsAccessKeyId");
- }
- if (secretAccessKey == null) {
- secretAccessKey = getConf().get("fs.s3.awsSecretAccessKey");
- }
- if (accessKey == null && secretAccessKey == null) {
- throw new IllegalArgumentException("AWS " +
- "Access Key ID and Secret Access Key " +
- "must be specified as the username " +
- "or password (respectively) of a s3 URL, " +
- "or by setting the " +
- "fs.s3.awsAccessKeyId or " +
- "fs.s3.awsSecretAccessKey properties (respectively).");
- } else if (accessKey == null) {
- throw new IllegalArgumentException("AWS " +
- "Access Key ID must be specified " +
- "as the username of a s3 URL, or by setting the " +
- "fs.s3.awsAccessKeyId property.");
- } else if (secretAccessKey == null) {
- throw new IllegalArgumentException("AWS " +
- "Secret Access Key must be specified " +
- "as the password of a s3 URL, or by setting the " +
- "fs.s3.awsSecretAccessKey property.");
- }
- AWSCredentials awsCredentials =
- new AWSCredentials(accessKey, secretAccessKey);
- this.s3Service = new RestS3Service(awsCredentials);
- } catch (S3ServiceException e) {
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- }
- bucket = new S3Bucket(uri.getHost());
- }
-
- private void migrate(Store oldStore, FileSystemStore newStore)
- throws IOException {
- for (Path path : oldStore.listAllPaths()) {
- INode inode = oldStore.retrieveINode(path);
- oldStore.deleteINode(path);
- newStore.storeINode(path, inode);
- }
- }
-
- private S3Object get(String key) {
- try {
- return s3Service.getObject(bucket.getName(), key);
- } catch (S3ServiceException e) {
- if ("NoSuchKey".equals(e.getS3ErrorCode())) {
- return null;
- }
- }
- return null;
- }
-
- interface Store {
-
- Set<Path> listAllPaths() throws IOException;
- INode retrieveINode(Path path) throws IOException;
- void deleteINode(Path path) throws IOException;
-
- }
-
- class UnversionedStore implements Store {
-
- @Override
- public Set<Path> listAllPaths() throws IOException {
- try {
- String prefix = urlEncode(Path.SEPARATOR);
- S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null);
- Set<Path> prefixes = new TreeSet<Path>();
- for (int i = 0; i < objects.length; i++) {
- prefixes.add(keyToPath(objects[i].getKey()));
- }
- return prefixes;
- } catch (S3ServiceException e) {
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- }
- }
-
- @Override
- public void deleteINode(Path path) throws IOException {
- delete(pathToKey(path));
- }
-
- private void delete(String key) throws IOException {
- try {
- s3Service.deleteObject(bucket, key);
- } catch (S3ServiceException e) {
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- }
- }
-
- @Override
- public INode retrieveINode(Path path) throws IOException {
- return INode.deserialize(get(pathToKey(path)));
- }
-
- private InputStream get(String key) throws IOException {
- try {
- S3Object object = s3Service.getObject(bucket.getName(), key);
- return object.getDataInputStream();
- } catch (S3ServiceException e) {
- if ("NoSuchKey".equals(e.getS3ErrorCode())) {
- return null;
- }
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- }
- throw new S3Exception(e);
- } catch (ServiceException e) {
- return null;
- }
- }
-
- private String pathToKey(Path path) {
- if (!path.isAbsolute()) {
- throw new IllegalArgumentException("Path must be absolute: " + path);
- }
- return urlEncode(path.toUri().getPath());
- }
-
- private Path keyToPath(String key) {
- return new Path(urlDecode(key));
- }
-
- private String urlEncode(String s) {
- try {
- return URLEncoder.encode(s, "UTF-8");
- } catch (UnsupportedEncodingException e) {
- // Should never happen since every implementation of the Java Platform
- // is required to support UTF-8.
- // See http://java.sun.com/j2se/1.5.0/docs/api/java/nio/charset/Charset.html
- throw new IllegalStateException(e);
- }
- }
-
- private String urlDecode(String s) {
- try {
- return URLDecoder.decode(s, "UTF-8");
- } catch (UnsupportedEncodingException e) {
- // Should never happen since every implementation of the Java Platform
- // is required to support UTF-8.
- // See http://java.sun.com/j2se/1.5.0/docs/api/java/nio/charset/Charset.html
- throw new IllegalStateException(e);
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3Credentials.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3Credentials.java
deleted file mode 100644
index 312bf65..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3Credentials.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.net.URI;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * <p>
- * Extracts AWS credentials from the filesystem URI or configuration.
- * </p>
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class S3Credentials {
-
- private String accessKey;
- private String secretAccessKey;
-
- /**
- * @throws IllegalArgumentException if credentials for S3 cannot be
- * determined.
- */
- public void initialize(URI uri, Configuration conf) {
- if (uri.getHost() == null) {
- throw new IllegalArgumentException("Invalid hostname in URI " + uri);
- }
-
- String userInfo = uri.getUserInfo();
- if (userInfo != null) {
- int index = userInfo.indexOf(':');
- if (index != -1) {
- accessKey = userInfo.substring(0, index);
- secretAccessKey = userInfo.substring(index + 1);
- } else {
- accessKey = userInfo;
- }
- }
-
- String scheme = uri.getScheme();
- String accessKeyProperty = String.format("fs.%s.awsAccessKeyId", scheme);
- String secretAccessKeyProperty =
- String.format("fs.%s.awsSecretAccessKey", scheme);
- if (accessKey == null) {
- accessKey = conf.get(accessKeyProperty);
- }
- if (secretAccessKey == null) {
- secretAccessKey = conf.get(secretAccessKeyProperty);
- }
- if (accessKey == null && secretAccessKey == null) {
- throw new IllegalArgumentException("AWS " +
- "Access Key ID and Secret Access " +
- "Key must be specified as the " +
- "username or password " +
- "(respectively) of a " + scheme +
- " URL, or by setting the " +
- accessKeyProperty + " or " +
- secretAccessKeyProperty +
- " properties (respectively).");
- } else if (accessKey == null) {
- throw new IllegalArgumentException("AWS " +
- "Access Key ID must be specified " +
- "as the username of a " + scheme +
- " URL, or by setting the " +
- accessKeyProperty + " property.");
- } else if (secretAccessKey == null) {
- throw new IllegalArgumentException("AWS " +
- "Secret Access Key must be " +
- "specified as the password of a " +
- scheme + " URL, or by setting the " +
- secretAccessKeyProperty +
- " property.");
- }
-
- }
-
- public String getAccessKey() {
- return accessKey;
- }
-
- public String getSecretAccessKey() {
- return secretAccessKey;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3Exception.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3Exception.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3Exception.java
deleted file mode 100644
index 4f07c4e..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3Exception.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Thrown if there is a problem communicating with Amazon S3.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class S3Exception extends IOException {
-
- private static final long serialVersionUID = 1L;
-
- public S3Exception(Throwable t) {
- super(t);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
deleted file mode 100644
index dda3cf6..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
+++ /dev/null
@@ -1,486 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.s3native.NativeS3FileSystem;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryProxy;
-import org.apache.hadoop.util.Progressable;
-
-/**
- * <p>
- * A block-based {@link FileSystem} backed by
- * <a href="http://aws.amazon.com/s3">Amazon S3</a>.
- * </p>
- * @see NativeS3FileSystem
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class S3FileSystem extends FileSystem {
-
- private URI uri;
-
- private FileSystemStore store;
-
- private Path workingDir;
-
- public S3FileSystem() {
- // set store in initialize()
- }
-
- public S3FileSystem(FileSystemStore store) {
- this.store = store;
- }
-
- /**
- * Return the protocol scheme for the FileSystem.
- * <p/>
- *
- * @return <code>s3</code>
- */
- @Override
- public String getScheme() {
- return "s3";
- }
-
- @Override
- public URI getUri() {
- return uri;
- }
-
- @Override
- public void initialize(URI uri, Configuration conf) throws IOException {
- super.initialize(uri, conf);
- if (store == null) {
- store = createDefaultStore(conf);
- }
- store.initialize(uri, conf);
- setConf(conf);
- this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
- this.workingDir =
- new Path("/user", System.getProperty("user.name")).makeQualified(this);
- }
-
- private static FileSystemStore createDefaultStore(Configuration conf) {
- FileSystemStore store = new Jets3tFileSystemStore();
-
- RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
- conf.getInt("fs.s3.maxRetries", 4),
- conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
- Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
- new HashMap<Class<? extends Exception>, RetryPolicy>();
- exceptionToPolicyMap.put(IOException.class, basePolicy);
- exceptionToPolicyMap.put(S3Exception.class, basePolicy);
-
- RetryPolicy methodPolicy = RetryPolicies.retryByException(
- RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
- Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
- methodNameToPolicyMap.put("storeBlock", methodPolicy);
- methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
-
- return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
- store, methodNameToPolicyMap);
- }
-
- @Override
- public Path getWorkingDirectory() {
- return workingDir;
- }
-
- @Override
- public void setWorkingDirectory(Path dir) {
- workingDir = makeAbsolute(dir);
- }
-
- private Path makeAbsolute(Path path) {
- if (path.isAbsolute()) {
- return path;
- }
- return new Path(workingDir, path);
- }
-
- /**
- * @param permission Currently ignored.
- */
- @Override
- public boolean mkdirs(Path path, FsPermission permission) throws IOException {
- Path absolutePath = makeAbsolute(path);
- List<Path> paths = new ArrayList<Path>();
- do {
- paths.add(0, absolutePath);
- absolutePath = absolutePath.getParent();
- } while (absolutePath != null);
-
- boolean result = true;
- for (Path p : paths) {
- result &= mkdir(p);
- }
- return result;
- }
-
- private boolean mkdir(Path path) throws IOException {
- Path absolutePath = makeAbsolute(path);
- INode inode = store.retrieveINode(absolutePath);
- if (inode == null) {
- store.storeINode(absolutePath, INode.DIRECTORY_INODE);
- } else if (inode.isFile()) {
- throw new IOException(String.format(
- "Can't make directory for path %s since it is a file.",
- absolutePath));
- }
- return true;
- }
-
- @Override
- public boolean isFile(Path path) throws IOException {
- INode inode = store.retrieveINode(makeAbsolute(path));
- if (inode == null) {
- return false;
- }
- return inode.isFile();
- }
-
- private INode checkFile(Path path) throws IOException {
- INode inode = store.retrieveINode(makeAbsolute(path));
- if (inode == null) {
- throw new IOException("No such file.");
- }
- if (inode.isDirectory()) {
- throw new IOException("Path " + path + " is a directory.");
- }
- return inode;
- }
-
- @Override
- public FileStatus[] listStatus(Path f) throws IOException {
- Path absolutePath = makeAbsolute(f);
- INode inode = store.retrieveINode(absolutePath);
- if (inode == null) {
- throw new FileNotFoundException("File " + f + " does not exist.");
- }
- if (inode.isFile()) {
- return new FileStatus[] {
- new S3FileStatus(f.makeQualified(this), inode)
- };
- }
- ArrayList<FileStatus> ret = new ArrayList<FileStatus>();
- for (Path p : store.listSubPaths(absolutePath)) {
- ret.add(getFileStatus(p.makeQualified(this)));
- }
- return ret.toArray(new FileStatus[0]);
- }
-
- /** This optional operation is not yet supported. */
- @Override
- public FSDataOutputStream append(Path f, int bufferSize,
- Progressable progress) throws IOException {
- throw new IOException("Not supported");
- }
-
- /**
- * @param permission Currently ignored.
- */
- @Override
- public FSDataOutputStream create(Path file, FsPermission permission,
- boolean overwrite, int bufferSize,
- short replication, long blockSize, Progressable progress)
- throws IOException {
-
- INode inode = store.retrieveINode(makeAbsolute(file));
- if (inode != null) {
- if (overwrite) {
- delete(file, true);
- } else {
- throw new FileAlreadyExistsException("File already exists: " + file);
- }
- } else {
- Path parent = file.getParent();
- if (parent != null) {
- if (!mkdirs(parent)) {
- throw new IOException("Mkdirs failed to create " + parent.toString());
- }
- }
- }
- return new FSDataOutputStream
- (new S3OutputStream(getConf(), store, makeAbsolute(file),
- blockSize, progress, bufferSize),
- statistics);
- }
-
- @Override
- public FSDataInputStream open(Path path, int bufferSize) throws IOException {
- INode inode = checkFile(path);
- return new FSDataInputStream(new S3InputStream(getConf(), store, inode,
- statistics));
- }
-
- @Override
- public boolean rename(Path src, Path dst) throws IOException {
- Path absoluteSrc = makeAbsolute(src);
- final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
- INode srcINode = store.retrieveINode(absoluteSrc);
- boolean debugEnabled = LOG.isDebugEnabled();
- if (srcINode == null) {
- // src path doesn't exist
- if (debugEnabled) {
- LOG.debug(debugPreamble + "returning false as src does not exist");
- }
- return false;
- }
-
- Path absoluteDst = makeAbsolute(dst);
-
- //validate the parent dir of the destination
- Path dstParent = absoluteDst.getParent();
- if (dstParent != null) {
- //if the dst parent is not root, make sure it exists
- INode dstParentINode = store.retrieveINode(dstParent);
- if (dstParentINode == null) {
- // dst parent doesn't exist
- if (debugEnabled) {
- LOG.debug(debugPreamble +
- "returning false as dst parent does not exist");
- }
- return false;
- }
- if (dstParentINode.isFile()) {
- // dst parent exists but is a file
- if (debugEnabled) {
- LOG.debug(debugPreamble +
- "returning false as dst parent exists and is a file");
- }
- return false;
- }
- }
-
- //get status of source
- boolean srcIsFile = srcINode.isFile();
-
- INode dstINode = store.retrieveINode(absoluteDst);
- boolean destExists = dstINode != null;
- boolean destIsDir = destExists && !dstINode.isFile();
- if (srcIsFile) {
-
- //source is a simple file
- if (destExists) {
- if (destIsDir) {
- //outcome #1 dest exists and is dir -filename to subdir of dest
- if (debugEnabled) {
- LOG.debug(debugPreamble +
- "copying src file under dest dir to " + absoluteDst);
- }
- absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
- } else {
- //outcome #2 dest it's a file: fail iff different from src
- boolean renamingOnToSelf = absoluteSrc.equals(absoluteDst);
- if (debugEnabled) {
- LOG.debug(debugPreamble +
- "copying file onto file, outcome is " + renamingOnToSelf);
- }
- return renamingOnToSelf;
- }
- } else {
- // #3 dest does not exist: use dest as path for rename
- if (debugEnabled) {
- LOG.debug(debugPreamble +
- "copying file onto file");
- }
- }
- } else {
- //here the source exists and is a directory
- // outcomes (given we know the parent dir exists if we get this far)
- // #1 destination is a file: fail
- // #2 destination is a directory: create a new dir under that one
- // #3 destination doesn't exist: create a new dir with that name
- // #3 and #4 are only allowed if the dest path is not == or under src
-
- if (destExists) {
- if (!destIsDir) {
- // #1 destination is a file: fail
- if (debugEnabled) {
- LOG.debug(debugPreamble +
- "returning false as src is a directory, but not dest");
- }
- return false;
- } else {
- // the destination dir exists
- // destination for rename becomes a subdir of the target name
- absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
- if (debugEnabled) {
- LOG.debug(debugPreamble +
- "copying src dir under dest dir to " + absoluteDst);
- }
- }
- }
- //the final destination directory is now know, so validate it for
- //illegal moves
-
- if (absoluteSrc.equals(absoluteDst)) {
- //you can't rename a directory onto itself
- if (debugEnabled) {
- LOG.debug(debugPreamble +
- "Dest==source && isDir -failing");
- }
- return false;
- }
- if (absoluteDst.toString().startsWith(absoluteSrc.toString() + "/")) {
- //you can't move a directory under itself
- if (debugEnabled) {
- LOG.debug(debugPreamble +
- "dst is equal to or under src dir -failing");
- }
- return false;
- }
- }
- //here the dest path is set up -so rename
- return renameRecursive(absoluteSrc, absoluteDst);
- }
-
- private boolean renameRecursive(Path src, Path dst) throws IOException {
- INode srcINode = store.retrieveINode(src);
- store.storeINode(dst, srcINode);
- store.deleteINode(src);
- if (srcINode.isDirectory()) {
- for (Path oldSrc : store.listDeepSubPaths(src)) {
- INode inode = store.retrieveINode(oldSrc);
- if (inode == null) {
- return false;
- }
- String oldSrcPath = oldSrc.toUri().getPath();
- String srcPath = src.toUri().getPath();
- String dstPath = dst.toUri().getPath();
- Path newDst = new Path(oldSrcPath.replaceFirst(srcPath, dstPath));
- store.storeINode(newDst, inode);
- store.deleteINode(oldSrc);
- }
- }
- return true;
- }
-
- @Override
- public boolean delete(Path path, boolean recursive) throws IOException {
- Path absolutePath = makeAbsolute(path);
- INode inode = store.retrieveINode(absolutePath);
- if (inode == null) {
- return false;
- }
- if (inode.isFile()) {
- store.deleteINode(absolutePath);
- for (Block block: inode.getBlocks()) {
- store.deleteBlock(block);
- }
- } else {
- FileStatus[] contents = null;
- try {
- contents = listStatus(absolutePath);
- } catch(FileNotFoundException fnfe) {
- return false;
- }
-
- if ((contents.length !=0) && (!recursive)) {
- throw new IOException("Directory " + path.toString()
- + " is not empty.");
- }
- for (FileStatus p:contents) {
- if (!delete(p.getPath(), recursive)) {
- return false;
- }
- }
- store.deleteINode(absolutePath);
- }
- return true;
- }
-
- /**
- * FileStatus for S3 file systems.
- */
- @Override
- public FileStatus getFileStatus(Path f) throws IOException {
- INode inode = store.retrieveINode(makeAbsolute(f));
- if (inode == null) {
- throw new FileNotFoundException(f + ": No such file or directory.");
- }
- return new S3FileStatus(f.makeQualified(this), inode);
- }
-
- @Override
- public long getDefaultBlockSize() {
- return getConf().getLong("fs.s3.block.size", 64 * 1024 * 1024);
- }
-
- @Override
- public String getCanonicalServiceName() {
- // Does not support Token
- return null;
- }
-
- // diagnostic methods
-
- void dump() throws IOException {
- store.dump();
- }
-
- void purge() throws IOException {
- store.purge();
- }
-
- private static class S3FileStatus extends FileStatus {
-
- S3FileStatus(Path f, INode inode) throws IOException {
- super(findLength(inode), inode.isDirectory(), 1,
- findBlocksize(inode), 0, f);
- }
-
- private static long findLength(INode inode) {
- if (!inode.isDirectory()) {
- long length = 0L;
- for (Block block : inode.getBlocks()) {
- length += block.getLength();
- }
- return length;
- }
- return 0;
- }
-
- private static long findBlocksize(INode inode) {
- final Block[] ret = inode.getBlocks();
- return ret == null ? 0L : ret[0].getLength();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemConfigKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemConfigKeys.java
deleted file mode 100644
index 8172a46..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemConfigKeys.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-
-/**
- * This class contains constants for configuration keys used
- * in the s3 file system.
- *
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class S3FileSystemConfigKeys extends CommonConfigurationKeys {
- public static final String S3_BLOCK_SIZE_KEY = "s3.blocksize";
- public static final long S3_BLOCK_SIZE_DEFAULT = 64*1024*1024;
- public static final String S3_REPLICATION_KEY = "s3.replication";
- public static final short S3_REPLICATION_DEFAULT = 1;
- public static final String S3_STREAM_BUFFER_SIZE_KEY =
- "s3.stream-buffer-size";
- public static final int S3_STREAM_BUFFER_SIZE_DEFAULT = 4096;
- public static final String S3_BYTES_PER_CHECKSUM_KEY =
- "s3.bytes-per-checksum";
- public static final int S3_BYTES_PER_CHECKSUM_DEFAULT = 512;
- public static final String S3_CLIENT_WRITE_PACKET_SIZE_KEY =
- "s3.client-write-packet-size";
- public static final int S3_CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024;
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemException.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemException.java
deleted file mode 100644
index cc1b463..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemException.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Thrown when there is a fatal exception while using {@link S3FileSystem}.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class S3FileSystemException extends IOException {
- private static final long serialVersionUID = 1L;
-
- public S3FileSystemException(String message) {
- super(message);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
deleted file mode 100644
index 5af57e6..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.fs.FileSystem;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-class S3InputStream extends FSInputStream {
-
- private FileSystemStore store;
-
- private Block[] blocks;
-
- private boolean closed;
-
- private long fileLength;
-
- private long pos = 0;
-
- private File blockFile;
-
- private DataInputStream blockStream;
-
- private long blockEnd = -1;
-
- private FileSystem.Statistics stats;
-
- private static final Log LOG =
- LogFactory.getLog(S3InputStream.class.getName());
-
-
- @Deprecated
- public S3InputStream(Configuration conf, FileSystemStore store,
- INode inode) {
- this(conf, store, inode, null);
- }
-
- public S3InputStream(Configuration conf, FileSystemStore store,
- INode inode, FileSystem.Statistics stats) {
-
- this.store = store;
- this.stats = stats;
- this.blocks = inode.getBlocks();
- for (Block block : blocks) {
- this.fileLength += block.getLength();
- }
- }
-
- @Override
- public synchronized long getPos() throws IOException {
- return pos;
- }
-
- @Override
- public synchronized int available() throws IOException {
- return (int) (fileLength - pos);
- }
-
- @Override
- public synchronized void seek(long targetPos) throws IOException {
- if (targetPos > fileLength) {
- throw new IOException("Cannot seek after EOF");
- }
- pos = targetPos;
- blockEnd = -1;
- }
-
- @Override
- public synchronized boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
-
- @Override
- public synchronized int read() throws IOException {
- if (closed) {
- throw new IOException("Stream closed");
- }
- int result = -1;
- if (pos < fileLength) {
- if (pos > blockEnd) {
- blockSeekTo(pos);
- }
- result = blockStream.read();
- if (result >= 0) {
- pos++;
- }
- }
- if (stats != null && result >= 0) {
- stats.incrementBytesRead(1);
- }
- return result;
- }
-
- @Override
- public synchronized int read(byte buf[], int off, int len) throws IOException {
- if (closed) {
- throw new IOException("Stream closed");
- }
- if (pos < fileLength) {
- if (pos > blockEnd) {
- blockSeekTo(pos);
- }
- int realLen = (int) Math.min((long) len, (blockEnd - pos + 1L));
- int result = blockStream.read(buf, off, realLen);
- if (result >= 0) {
- pos += result;
- }
- if (stats != null && result > 0) {
- stats.incrementBytesRead(result);
- }
- return result;
- }
- return -1;
- }
-
- private synchronized void blockSeekTo(long target) throws IOException {
- //
- // Compute desired block
- //
- int targetBlock = -1;
- long targetBlockStart = 0;
- long targetBlockEnd = 0;
- for (int i = 0; i < blocks.length; i++) {
- long blockLength = blocks[i].getLength();
- targetBlockEnd = targetBlockStart + blockLength - 1;
-
- if (target >= targetBlockStart && target <= targetBlockEnd) {
- targetBlock = i;
- break;
- } else {
- targetBlockStart = targetBlockEnd + 1;
- }
- }
- if (targetBlock < 0) {
- throw new IOException(
- "Impossible situation: could not find target position " + target);
- }
- long offsetIntoBlock = target - targetBlockStart;
-
- // read block blocks[targetBlock] from position offsetIntoBlock
-
- this.blockFile = store.retrieveBlock(blocks[targetBlock], offsetIntoBlock);
-
- this.pos = target;
- this.blockEnd = targetBlockEnd;
- this.blockStream = new DataInputStream(new FileInputStream(blockFile));
-
- }
-
- @Override
- public void close() throws IOException {
- if (closed) {
- return;
- }
- if (blockStream != null) {
- blockStream.close();
- blockStream = null;
- }
- if (blockFile != null) {
- boolean b = blockFile.delete();
- if (!b) {
- LOG.warn("Ignoring failed delete");
- }
- }
- super.close();
- closed = true;
- }
-
- /**
- * We don't support marks.
- */
- @Override
- public boolean markSupported() {
- return false;
- }
-
- @Override
- public void mark(int readLimit) {
- // Do nothing
- }
-
- @Override
- public void reset() throws IOException {
- throw new IOException("Mark not supported");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3OutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3OutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3OutputStream.java
deleted file mode 100644
index 761f2ce..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3OutputStream.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3.INode.FileType;
-import org.apache.hadoop.util.Progressable;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-class S3OutputStream extends OutputStream {
-
- private Configuration conf;
-
- private int bufferSize;
-
- private FileSystemStore store;
-
- private Path path;
-
- private long blockSize;
-
- private File backupFile;
-
- private OutputStream backupStream;
-
- private Random r = new Random();
-
- private boolean closed;
-
- private int pos = 0;
-
- private long filePos = 0;
-
- private int bytesWrittenToBlock = 0;
-
- private byte[] outBuf;
-
- private List<Block> blocks = new ArrayList<Block>();
-
- private Block nextBlock;
-
- private static final Log LOG =
- LogFactory.getLog(S3OutputStream.class.getName());
-
-
- public S3OutputStream(Configuration conf, FileSystemStore store,
- Path path, long blockSize, Progressable progress,
- int buffersize) throws IOException {
-
- this.conf = conf;
- this.store = store;
- this.path = path;
- this.blockSize = blockSize;
- this.backupFile = newBackupFile();
- this.backupStream = new FileOutputStream(backupFile);
- this.bufferSize = buffersize;
- this.outBuf = new byte[bufferSize];
-
- }
-
- private File newBackupFile() throws IOException {
- File dir = new File(conf.get("fs.s3.buffer.dir"));
- if (!dir.exists() && !dir.mkdirs()) {
- throw new IOException("Cannot create S3 buffer directory: " + dir);
- }
- File result = File.createTempFile("output-", ".tmp", dir);
- result.deleteOnExit();
- return result;
- }
-
- public long getPos() throws IOException {
- return filePos;
- }
-
- @Override
- public synchronized void write(int b) throws IOException {
- if (closed) {
- throw new IOException("Stream closed");
- }
-
- if ((bytesWrittenToBlock + pos == blockSize) || (pos >= bufferSize)) {
- flush();
- }
- outBuf[pos++] = (byte) b;
- filePos++;
- }
-
- @Override
- public synchronized void write(byte b[], int off, int len) throws IOException {
- if (closed) {
- throw new IOException("Stream closed");
- }
- while (len > 0) {
- int remaining = bufferSize - pos;
- int toWrite = Math.min(remaining, len);
- System.arraycopy(b, off, outBuf, pos, toWrite);
- pos += toWrite;
- off += toWrite;
- len -= toWrite;
- filePos += toWrite;
-
- if ((bytesWrittenToBlock + pos >= blockSize) || (pos == bufferSize)) {
- flush();
- }
- }
- }
-
- @Override
- public synchronized void flush() throws IOException {
- if (closed) {
- throw new IOException("Stream closed");
- }
-
- if (bytesWrittenToBlock + pos >= blockSize) {
- flushData((int) blockSize - bytesWrittenToBlock);
- }
- if (bytesWrittenToBlock == blockSize) {
- endBlock();
- }
- flushData(pos);
- }
-
- private synchronized void flushData(int maxPos) throws IOException {
- int workingPos = Math.min(pos, maxPos);
-
- if (workingPos > 0) {
- //
- // To the local block backup, write just the bytes
- //
- backupStream.write(outBuf, 0, workingPos);
-
- //
- // Track position
- //
- bytesWrittenToBlock += workingPos;
- System.arraycopy(outBuf, workingPos, outBuf, 0, pos - workingPos);
- pos -= workingPos;
- }
- }
-
- private synchronized void endBlock() throws IOException {
- //
- // Done with local copy
- //
- backupStream.close();
-
- //
- // Send it to S3
- //
- // TODO: Use passed in Progressable to report progress.
- nextBlockOutputStream();
- store.storeBlock(nextBlock, backupFile);
- internalClose();
-
- //
- // Delete local backup, start new one
- //
- boolean b = backupFile.delete();
- if (!b) {
- LOG.warn("Ignoring failed delete");
- }
- backupFile = newBackupFile();
- backupStream = new FileOutputStream(backupFile);
- bytesWrittenToBlock = 0;
- }
-
- private synchronized void nextBlockOutputStream() throws IOException {
- long blockId = r.nextLong();
- while (store.blockExists(blockId)) {
- blockId = r.nextLong();
- }
- nextBlock = new Block(blockId, bytesWrittenToBlock);
- blocks.add(nextBlock);
- bytesWrittenToBlock = 0;
- }
-
- private synchronized void internalClose() throws IOException {
- INode inode = new INode(FileType.FILE, blocks.toArray(new Block[blocks
- .size()]));
- store.storeINode(path, inode);
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (closed) {
- return;
- }
-
- flush();
- if (filePos == 0 || bytesWrittenToBlock != 0) {
- endBlock();
- }
-
- backupStream.close();
- boolean b = backupFile.delete();
- if (!b) {
- LOG.warn("Ignoring failed delete");
- }
-
- super.close();
-
- closed = true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/VersionMismatchException.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/VersionMismatchException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/VersionMismatchException.java
deleted file mode 100644
index ccc8969..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/VersionMismatchException.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Thrown when Hadoop cannot read the version of the data stored
- * in {@link S3FileSystem}.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class VersionMismatchException extends S3FileSystemException {
- private static final long serialVersionUID = 1L;
-
- public VersionMismatchException(String clientVersion, String dataVersion) {
- super("Version mismatch: client expects version " + clientVersion +
- ", but data has version " +
- (dataVersion == null ? "[unversioned]" : dataVersion));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/package.html
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/package.html b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/package.html
deleted file mode 100644
index dd601e1..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/package.html
+++ /dev/null
@@ -1,55 +0,0 @@
-<html>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<body>
-
-<p>A distributed, block-based implementation of {@link
-org.apache.hadoop.fs.FileSystem} that uses <a href="http://aws.amazon.com/s3">Amazon S3</a>
-as a backing store.</p>
-
-<p>
-Files are stored in S3 as blocks (represented by
-{@link org.apache.hadoop.fs.s3.Block}), which have an ID and a length.
-Block metadata is stored in S3 as a small record (represented by
-{@link org.apache.hadoop.fs.s3.INode}) using the URL-encoded
-path string as a key. Inodes record the file type (regular file or directory) and the list of blocks.
-This design makes it easy to seek to any given position in a file by reading the inode data to compute
-which block to access, then using S3's support for
-<a href="http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.2">HTTP Range</a> headers
-to start streaming from the correct position.
-Renames are also efficient since only the inode is moved (by a DELETE followed by a PUT since
-S3 does not support renames).
-</p>
-<p>
-For a single file <i>/dir1/file1</i> which takes two blocks of storage, the file structure in S3
-would be something like this:
-</p>
-<pre>
-/
-/dir1
-/dir1/file1
-block-6415776850131549260
-block-3026438247347758425
-</pre>
-<p>
-Inodes start with a leading <code>/</code>, while blocks are prefixed with <code>block-</code>.
-</p>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/FileMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/FileMetadata.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/FileMetadata.java
deleted file mode 100644
index 2746af4..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/FileMetadata.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * <p>
- * Holds basic metadata for a file stored in a {@link NativeFileSystemStore}.
- * </p>
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-class FileMetadata {
- private final String key;
- private final long length;
- private final long lastModified;
-
- public FileMetadata(String key, long length, long lastModified) {
- this.key = key;
- this.length = length;
- this.lastModified = lastModified;
- }
-
- public String getKey() {
- return key;
- }
-
- public long getLength() {
- return length;
- }
-
- public long getLastModified() {
- return lastModified;
- }
-
- @Override
- public String toString() {
- return "FileMetadata[" + key + ", " + length + ", " + lastModified + "]";
- }
-
-}
[10/20] git commit: MAPREDUCE-6075. HistoryServerFileSystemStateStore
can create zero-length files. Contributed by Jason Lowe
Posted by sz...@apache.org.
MAPREDUCE-6075. HistoryServerFileSystemStateStore can create zero-length files. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7f80e142
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7f80e142
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7f80e142
Branch: refs/heads/HDFS-6584
Commit: 7f80e142096ab21b4852635a127589e34a747bc4
Parents: 7d38ffc
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Sep 10 22:10:36 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Sep 10 22:22:39 2014 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../v2/hs/HistoryServerFileSystemStateStoreService.java | 6 ++++++
2 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f80e142/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a5e5920..490c0a1 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -276,6 +276,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6071. JobImpl#makeUberDecision doesn't log that Uber mode is
disabled because of too much CPUs (Tsuyoshi OZAWA via jlowe)
+ MAPREDUCE-6075. HistoryServerFileSystemStateStore can create zero-length
+ files (jlowe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f80e142/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
index dcea333..9902f5e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
@@ -189,6 +189,8 @@ public class HistoryServerFileSystemStateStoreService
DataOutputStream dataStream = new DataOutputStream(memStream);
try {
key.write(dataStream);
+ dataStream.close();
+ dataStream = null;
} finally {
IOUtils.cleanup(LOG, dataStream);
}
@@ -260,6 +262,8 @@ public class HistoryServerFileSystemStateStoreService
try {
try {
out.write(data);
+ out.close();
+ out = null;
} finally {
IOUtils.cleanup(LOG, out);
}
@@ -299,6 +303,8 @@ public class HistoryServerFileSystemStateStoreService
try {
tokenId.write(dataStream);
dataStream.writeLong(renewDate);
+ dataStream.close();
+ dataStream = null;
} finally {
IOUtils.cleanup(LOG, dataStream);
}
[17/20] YARN-415. Capture aggregate memory allocation at the
app-level for chargeback. Contributed by Eric Payne & Andrey Klochkov
Posted by sz...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
new file mode 100644
index 0000000..d4ecc89
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.commons.lang.time.DateUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestContainerResourceUsage {
+
+ private YarnConfiguration conf;
+
+ @Before
+ public void setup() throws UnknownHostException {
+ Logger rootLogger = LogManager.getRootLogger();
+ rootLogger.setLevel(Level.DEBUG);
+ conf = new YarnConfiguration();
+ UserGroupInformation.setConfiguration(conf);
+ conf.set(YarnConfiguration.RECOVERY_ENABLED, "true");
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
+ }
+
+ @After
+ public void tearDown() {
+ }
+
+ @Test (timeout = 60000)
+ public void testUsageWithOneAttemptAndOneContainer() throws Exception {
+ MockRM rm = new MockRM(conf);
+ rm.start();
+
+ MockNM nm =
+ new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
+ nm.registerNode();
+
+ RMApp app0 = rm.submitApp(200);
+
+ RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
+ Assert.assertTrue(
+ "Before app submittion, memory seconds should have been 0 but was "
+ + rmAppMetrics.getMemorySeconds(),
+ rmAppMetrics.getMemorySeconds() == 0);
+ Assert.assertTrue(
+ "Before app submission, vcore seconds should have been 0 but was "
+ + rmAppMetrics.getVcoreSeconds(),
+ rmAppMetrics.getVcoreSeconds() == 0);
+
+ RMAppAttempt attempt0 = app0.getCurrentAppAttempt();
+
+ nm.nodeHeartbeat(true);
+ MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId());
+ am0.registerAppAttempt();
+
+ RMContainer rmContainer =
+ rm.getResourceScheduler()
+ .getRMContainer(attempt0.getMasterContainer().getId());
+
+ // Allow metrics to accumulate.
+ Thread.sleep(1000);
+
+ rmAppMetrics = app0.getRMAppMetrics();
+ Assert.assertTrue(
+ "While app is running, memory seconds should be >0 but is "
+ + rmAppMetrics.getMemorySeconds(),
+ rmAppMetrics.getMemorySeconds() > 0);
+ Assert.assertTrue(
+ "While app is running, vcore seconds should be >0 but is "
+ + rmAppMetrics.getVcoreSeconds(),
+ rmAppMetrics.getVcoreSeconds() > 0);
+
+ MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);
+
+ AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
+ rmAppMetrics = app0.getRMAppMetrics();
+
+ Assert.assertEquals("Unexcpected MemorySeconds value",
+ ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
+ Assert.assertEquals("Unexpected VcoreSeconds value",
+ ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
+
+ rm.stop();
+ }
+
+ @Test (timeout = 60000)
+ public void testUsageWithMultipleContainersAndRMRestart() throws Exception {
+ // Set max attempts to 1 so that when the first attempt fails, the app
+ // won't try to start a new one.
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+
+ MockRM rm0 = new MockRM(conf, memStore);
+ rm0.start();
+ MockNM nm =
+ new MockNM("127.0.0.1:1234", 65536, rm0.getResourceTrackerService());
+ nm.registerNode();
+
+ RMApp app0 = rm0.submitApp(200);
+
+ rm0.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED);
+ RMAppAttempt attempt0 = app0.getCurrentAppAttempt();
+ ApplicationAttemptId attemptId0 = attempt0.getAppAttemptId();
+ rm0.waitForState(attemptId0, RMAppAttemptState.SCHEDULED);
+
+ nm.nodeHeartbeat(true);
+ rm0.waitForState(attemptId0, RMAppAttemptState.ALLOCATED);
+ MockAM am0 = rm0.sendAMLaunched(attempt0.getAppAttemptId());
+ am0.registerAppAttempt();
+
+ int NUM_CONTAINERS = 2;
+ am0.allocate("127.0.0.1" , 1000, NUM_CONTAINERS,
+ new ArrayList<ContainerId>());
+ nm.nodeHeartbeat(true);
+ List<Container> conts = am0.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ while (conts.size() != NUM_CONTAINERS) {
+ nm.nodeHeartbeat(true);
+ conts.addAll(am0.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers());
+ Thread.sleep(500);
+ }
+
+ // launch the 2nd and 3rd containers.
+ for (Container c : conts) {
+ nm.nodeHeartbeat(attempt0.getAppAttemptId(),
+ c.getId().getId(), ContainerState.RUNNING);
+ rm0.waitForState(nm, c.getId(), RMContainerState.RUNNING);
+ }
+
+ // Get the RMContainers for all of the live containers, to be used later
+ // for metrics calculations and comparisons.
+ Collection<RMContainer> rmContainers =
+ rm0.scheduler
+ .getSchedulerAppInfo(attempt0.getAppAttemptId())
+ .getLiveContainers();
+
+ // Give the metrics time to accumulate.
+ Thread.sleep(1000);
+
+ // Stop all non-AM containers
+ for (Container c : conts) {
+ if (c.getId().getId() == 1) continue;
+ nm.nodeHeartbeat(attempt0.getAppAttemptId(),
+ c.getId().getId(), ContainerState.COMPLETE);
+ rm0.waitForState(nm, c.getId(), RMContainerState.COMPLETED);
+ }
+
+ // After all other containers have completed, manually complete the master
+ // container in order to trigger a save to the state store of the resource
+ // usage metrics. This will cause the attempt to fail, and, since the max
+ // attempt retries is 1, the app will also fail. This is intentional so
+ // that all containers will complete prior to saving.
+ ContainerId cId = ContainerId.newInstance(attempt0.getAppAttemptId(), 1);
+ nm.nodeHeartbeat(attempt0.getAppAttemptId(),
+ cId.getId(), ContainerState.COMPLETE);
+ rm0.waitForState(nm, cId, RMContainerState.COMPLETED);
+
+ // Check that the container metrics match those from the app usage report.
+ long memorySeconds = 0;
+ long vcoreSeconds = 0;
+ for (RMContainer c : rmContainers) {
+ AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
+ memorySeconds += ru.getMemorySeconds();
+ vcoreSeconds += ru.getVcoreSeconds();
+ }
+
+ RMAppMetrics metricsBefore = app0.getRMAppMetrics();
+ Assert.assertEquals("Unexcpected MemorySeconds value",
+ memorySeconds, metricsBefore.getMemorySeconds());
+ Assert.assertEquals("Unexpected VcoreSeconds value",
+ vcoreSeconds, metricsBefore.getVcoreSeconds());
+
+ // create new RM to represent RM restart. Load up the state store.
+ MockRM rm1 = new MockRM(conf, memStore);
+ rm1.start();
+ RMApp app0After =
+ rm1.getRMContext().getRMApps().get(app0.getApplicationId());
+
+ // Compare container resource usage metrics from before and after restart.
+ RMAppMetrics metricsAfter = app0After.getRMAppMetrics();
+ Assert.assertEquals("Vcore seconds were not the same after RM Restart",
+ metricsBefore.getVcoreSeconds(), metricsAfter.getVcoreSeconds());
+ Assert.assertEquals("Memory seconds were not the same after RM Restart",
+ metricsBefore.getMemorySeconds(), metricsAfter.getMemorySeconds());
+
+ rm0.stop();
+ rm0.close();
+ rm1.stop();
+ rm1.close();
+ }
+
+ @Test(timeout = 60000)
+ public void testUsageAfterAMRestartWithMultipleContainers() throws Exception {
+ amRestartTests(false);
+ }
+
+ @Test(timeout = 60000)
+ public void testUsageAfterAMRestartKeepContainers() throws Exception {
+ amRestartTests(true);
+ }
+
+ private void amRestartTests(boolean keepRunningContainers)
+ throws Exception {
+ MockRM rm = new MockRM(conf);
+ rm.start();
+
+ RMApp app =
+ rm.submitApp(200, "name", "user",
+ new HashMap<ApplicationAccessType, String>(), false, "default", -1,
+ null, "MAPREDUCE", false, keepRunningContainers);
+ MockNM nm =
+ new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService());
+ nm.registerNode();
+
+ MockAM am0 = MockRM.launchAndRegisterAM(app, rm, nm);
+ int NUM_CONTAINERS = 1;
+ // allocate NUM_CONTAINERS containers
+ am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
+ new ArrayList<ContainerId>());
+ nm.nodeHeartbeat(true);
+
+ // wait for containers to be allocated.
+ List<Container> containers =
+ am0.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ while (containers.size() != NUM_CONTAINERS) {
+ nm.nodeHeartbeat(true);
+ containers.addAll(am0.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers());
+ Thread.sleep(200);
+ }
+
+ // launch the 2nd container.
+ ContainerId containerId2 =
+ ContainerId.newInstance(am0.getApplicationAttemptId(), 2);
+ nm.nodeHeartbeat(am0.getApplicationAttemptId(),
+ containerId2.getId(), ContainerState.RUNNING);
+ rm.waitForState(nm, containerId2, RMContainerState.RUNNING);
+
+ // Capture the containers here so the metrics can be calculated after the
+ // app has completed.
+ Collection<RMContainer> rmContainers =
+ rm.scheduler
+ .getSchedulerAppInfo(am0.getApplicationAttemptId())
+ .getLiveContainers();
+
+ // fail the first app attempt by sending CONTAINER_FINISHED event without
+ // registering.
+ ContainerId amContainerId =
+ app.getCurrentAppAttempt().getMasterContainer().getId();
+ nm.nodeHeartbeat(am0.getApplicationAttemptId(),
+ amContainerId.getId(), ContainerState.COMPLETE);
+ am0.waitForState(RMAppAttemptState.FAILED);
+
+ long memorySeconds = 0;
+ long vcoreSeconds = 0;
+
+ // Calculate container usage metrics for first attempt.
+ if (keepRunningContainers) {
+ // Only calculate the usage for the one container that has completed.
+ for (RMContainer c : rmContainers) {
+ if (c.getContainerId().equals(amContainerId)) {
+ AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
+ memorySeconds += ru.getMemorySeconds();
+ vcoreSeconds += ru.getVcoreSeconds();
+ } else {
+ // The remaining container should be RUNNING.
+ Assert.assertTrue("After first attempt failed, remaining container "
+ + "should still be running. ",
+ c.getContainerState().equals(ContainerState.RUNNING));
+ }
+ }
+ } else {
+ // If keepRunningContainers is false, all live containers should now
+ // be completed. Calculate the resource usage metrics for all of them.
+ for (RMContainer c : rmContainers) {
+ AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
+ memorySeconds += ru.getMemorySeconds();
+ vcoreSeconds += ru.getVcoreSeconds();
+ }
+ }
+
+ // wait for app to start a new attempt.
+ rm.waitForState(app.getApplicationId(), RMAppState.ACCEPTED);
+
+ // assert this is a new AM.
+ RMAppAttempt attempt2 = app.getCurrentAppAttempt();
+ Assert.assertFalse(attempt2.getAppAttemptId()
+ .equals(am0.getApplicationAttemptId()));
+
+ // launch the new AM
+ nm.nodeHeartbeat(true);
+ MockAM am1 = rm.sendAMLaunched(attempt2.getAppAttemptId());
+ am1.registerAppAttempt();
+
+ // allocate NUM_CONTAINERS containers
+ am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
+ new ArrayList<ContainerId>());
+ nm.nodeHeartbeat(true);
+
+ // wait for containers to be allocated.
+ containers =
+ am1.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ while (containers.size() != NUM_CONTAINERS) {
+ nm.nodeHeartbeat(true);
+ containers.addAll(am1.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers());
+ Thread.sleep(200);
+ }
+
+ rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
+
+ // Capture running containers for later use by metrics calculations.
+ rmContainers = rm.scheduler.getSchedulerAppInfo(attempt2.getAppAttemptId())
+ .getLiveContainers();
+
+ // complete container by sending the container complete event which has
+ // earlier attempt's attemptId
+ amContainerId = app.getCurrentAppAttempt().getMasterContainer().getId();
+ nm.nodeHeartbeat(am0.getApplicationAttemptId(),
+ amContainerId.getId(), ContainerState.COMPLETE);
+
+ MockRM.finishAMAndVerifyAppState(app, rm, nm, am1);
+
+ // Calculate container usage metrics for second attempt.
+ for (RMContainer c : rmContainers) {
+ AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
+ memorySeconds += ru.getMemorySeconds();
+ vcoreSeconds += ru.getVcoreSeconds();
+ }
+
+ RMAppMetrics rmAppMetrics = app.getRMAppMetrics();
+
+ Assert.assertEquals("Unexcpected MemorySeconds value",
+ memorySeconds, rmAppMetrics.getMemorySeconds());
+ Assert.assertEquals("Unexpected VcoreSeconds value",
+ vcoreSeconds, rmAppMetrics.getVcoreSeconds());
+
+ rm.stop();
+ return;
+ }
+
+ private AggregateAppResourceUsage calculateContainerResourceMetrics(
+ RMContainer rmContainer) {
+ Resource resource = rmContainer.getContainer().getResource();
+ long usedMillis =
+ rmContainer.getFinishTime() - rmContainer.getCreationTime();
+ long memorySeconds = resource.getMemory()
+ * usedMillis / DateUtils.MILLIS_PER_SECOND;
+ long vcoreSeconds = resource.getVirtualCores()
+ * usedMillis / DateUtils.MILLIS_PER_SECOND;
+ return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index ff60fcd..15e45c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -170,7 +171,7 @@ public abstract class MockAsm extends MockApps {
@Override
public RMAppMetrics getRMAppMetrics() {
- return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0);
+ return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
}
}
@@ -259,6 +260,22 @@ public abstract class MockAsm extends MockApps {
public Set<String> getApplicationTags() {
return null;
}
+
+ @Override
+ public ApplicationReport createAndGetApplicationReport(
+ String clientUserName, boolean allowAccess) {
+ ApplicationResourceUsageReport usageReport =
+ ApplicationResourceUsageReport.newInstance(0, 0, null, null, null,
+ 0, 0);
+ ApplicationReport report = ApplicationReport.newInstance(
+ getApplicationId(), appAttemptId, getUser(), getQueue(),
+ getName(), null, 0, null, null, getDiagnostics().toString(),
+ getTrackingUrl(), getStartTime(), getFinishTime(),
+ getFinalApplicationStatus(), usageReport , null, getProgress(),
+ type, null);
+ return report;
+ }
+
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index 620ba9f..2621dff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -64,7 +64,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMSta
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
@@ -152,6 +154,8 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{
SecretKey clientTokenMasterKey, TestDispatcher dispatcher)
throws Exception {
+ RMAppAttemptMetrics mockRmAppAttemptMetrics =
+ mock(RMAppAttemptMetrics.class);
Container container = new ContainerPBImpl();
container.setId(ConverterUtils.toContainerId(containerIdStr));
RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
@@ -160,6 +164,10 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{
when(mockAttempt.getAMRMToken()).thenReturn(appToken);
when(mockAttempt.getClientTokenMasterKey())
.thenReturn(clientTokenMasterKey);
+ when(mockAttempt.getRMAppAttemptMetrics())
+ .thenReturn(mockRmAppAttemptMetrics);
+ when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
+ .thenReturn(new AggregateAppResourceUsage(0,0));
dispatcher.attemptId = attemptId;
store.storeNewApplicationAttempt(mockAttempt);
waitNotify(dispatcher);
@@ -224,6 +232,8 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{
"container_1352994193343_0002_01_000001", null, null, dispatcher);
RMApp mockRemovedApp = mock(RMApp.class);
+ RMAppAttemptMetrics mockRmAppAttemptMetrics =
+ mock(RMAppAttemptMetrics.class);
HashMap<ApplicationAttemptId, RMAppAttempt> attempts =
new HashMap<ApplicationAttemptId, RMAppAttempt>();
ApplicationSubmissionContext context =
@@ -234,6 +244,10 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{
when(mockRemovedApp.getAppAttempts()).thenReturn(attempts);
RMAppAttempt mockRemovedAttempt = mock(RMAppAttempt.class);
when(mockRemovedAttempt.getAppAttemptId()).thenReturn(attemptIdRemoved);
+ when(mockRemovedAttempt.getRMAppAttemptMetrics())
+ .thenReturn(mockRmAppAttemptMetrics);
+ when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
+ .thenReturn(new AggregateAppResourceUsage(0,0));
attempts.put(attemptIdRemoved, mockRemovedAttempt);
store.removeApplication(mockRemovedApp);
@@ -304,7 +318,7 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{
oldAttemptState.getAppAttemptCredentials(),
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
- FinalApplicationStatus.SUCCEEDED, 100);
+ FinalApplicationStatus.SUCCEEDED, 100, 0, 0);
store.updateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
@@ -327,7 +341,7 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{
oldAttemptState.getAppAttemptCredentials(),
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
- FinalApplicationStatus.SUCCEEDED, 111);
+ FinalApplicationStatus.SUCCEEDED, 111, 0, 0);
store.updateApplicationAttemptState(dummyAttempt);
// let things settle down
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 3c871df..5874b5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -19,8 +19,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -38,6 +40,7 @@ import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.MockApps;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -61,6 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
@@ -74,6 +78,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.mockito.ArgumentCaptor;
+import org.mockito.Matchers;
@RunWith(value = Parameterized.class)
@@ -189,7 +194,7 @@ public class TestRMAppTransitions {
AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class);
store = mock(RMStateStore.class);
writer = mock(RMApplicationHistoryWriter.class);
- this.rmContext =
+ RMContext realRMContext =
new RMContextImpl(rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
null, new AMRMTokenSecretManager(conf, this.rmContext),
@@ -197,7 +202,14 @@ public class TestRMAppTransitions {
new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM(),
writer);
- ((RMContextImpl)rmContext).setStateStore(store);
+ ((RMContextImpl)realRMContext).setStateStore(store);
+
+ this.rmContext = spy(realRMContext);
+
+ ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
+ doReturn(null).when(resourceScheduler)
+ .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
+ doReturn(resourceScheduler).when(rmContext).getScheduler();
rmDispatcher.register(RMAppAttemptEventType.class,
new TestApplicationAttemptEventDispatcher(this.rmContext));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index efcecd9..ae318b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
@@ -49,6 +50,7 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -87,6 +89,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
@@ -107,6 +111,8 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.mockito.ArgumentCaptor;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
@RunWith(value = Parameterized.class)
public class TestRMAppAttemptTransitions {
@@ -120,7 +126,9 @@ public class TestRMAppAttemptTransitions {
private boolean isSecurityEnabled;
private RMContext rmContext;
+ private RMContext spyRMContext;
private YarnScheduler scheduler;
+ private ResourceScheduler resourceScheduler;
private ApplicationMasterService masterService;
private ApplicationMasterLauncher applicationMasterLauncher;
private AMLivelinessMonitor amLivelinessMonitor;
@@ -262,7 +270,20 @@ public class TestRMAppAttemptTransitions {
ApplicationId applicationId = MockApps.newAppID(appId++);
ApplicationAttemptId applicationAttemptId =
ApplicationAttemptId.newInstance(applicationId, 0);
-
+
+ resourceScheduler = mock(ResourceScheduler.class);
+
+ ApplicationResourceUsageReport appResUsgRpt =
+ mock(ApplicationResourceUsageReport.class);
+ when(appResUsgRpt.getMemorySeconds()).thenReturn(0L);
+ when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L);
+ when(resourceScheduler
+ .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()))
+ .thenReturn(appResUsgRpt);
+ spyRMContext = spy(rmContext);
+ Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler();
+
+
final String user = MockApps.newUserName();
final String queue = MockApps.newQueue();
submissionContext = mock(ApplicationSubmissionContext.class);
@@ -278,17 +299,18 @@ public class TestRMAppAttemptTransitions {
application = mock(RMAppImpl.class);
applicationAttempt =
- new RMAppAttemptImpl(applicationAttemptId, rmContext, scheduler,
+ new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler,
masterService, submissionContext, new Configuration(), false);
when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
when(application.getApplicationId()).thenReturn(applicationId);
-
+ spyRMContext.getRMApps().put(application.getApplicationId(), application);
+
testAppAttemptNewState();
}
@After
public void tearDown() throws Exception {
- ((AsyncDispatcher)this.rmContext.getDispatcher()).stop();
+ ((AsyncDispatcher)this.spyRMContext.getDispatcher()).stop();
}
@@ -699,6 +721,46 @@ public class TestRMAppAttemptTransitions {
}
@Test
+ public void testUsageReport() {
+ // scheduler has info on running apps
+ ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
+ ApplicationResourceUsageReport appResUsgRpt =
+ mock(ApplicationResourceUsageReport.class);
+ when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
+ when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
+ when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class)))
+ .thenReturn(appResUsgRpt);
+
+ // start and finish the attempt
+ Container amContainer = allocateApplicationAttempt();
+ launchApplicationAttempt(amContainer);
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
+ applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(attemptId,
+ "", FinalApplicationStatus.SUCCEEDED, ""));
+
+ // expect usage stats to come from the scheduler report
+ ApplicationResourceUsageReport report =
+ applicationAttempt.getApplicationResourceUsageReport();
+ Assert.assertEquals(123456L, report.getMemorySeconds());
+ Assert.assertEquals(55544L, report.getVcoreSeconds());
+
+ // finish app attempt and remove it from scheduler
+ when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
+ when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
+ sendAttemptUpdateSavedEvent(applicationAttempt);
+ applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
+ attemptId,
+ ContainerStatus.newInstance(
+ amContainer.getId(), ContainerState.COMPLETE, "", 0)));
+
+ when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);
+
+ report = applicationAttempt.getApplicationResourceUsageReport();
+ Assert.assertEquals(223456, report.getMemorySeconds());
+ Assert.assertEquals(75544, report.getVcoreSeconds());
+ }
+
+ @Test
public void testUnmanagedAMUnexpectedRegistration() {
unmanagedAM = true;
when(submissionContext.getUnmanagedAM()).thenReturn(true);
@@ -1243,7 +1305,7 @@ public class TestRMAppAttemptTransitions {
public void testContainersCleanupForLastAttempt() {
// create a failed attempt.
applicationAttempt =
- new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), rmContext,
+ new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext,
scheduler, masterService, submissionContext, new Configuration(),
true);
when(submissionContext.getKeepContainersAcrossApplicationAttempts())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index 44f8381..9862cfe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -23,10 +23,13 @@ import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -59,6 +62,8 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
@SuppressWarnings({ "unchecked", "rawtypes" })
public class TestRMContainerImpl {
@@ -86,12 +91,18 @@ public class TestRMContainerImpl {
Container container = BuilderUtils.newContainer(containerId, nodeId,
"host:3465", resource, priority, null);
+ ConcurrentMap<ApplicationId, RMApp> rmApps =
+ spy(new ConcurrentHashMap<ApplicationId, RMApp>());
+ RMApp rmApp = mock(RMApp.class);
+ when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
+ Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any());
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
RMContext rmContext = mock(RMContext.class);
when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
+ when(rmContext.getRMApps()).thenReturn(rmApps);
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
nodeId, "user", rmContext);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index d5eb933..083cb71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -39,12 +39,15 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -59,6 +62,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -79,6 +83,8 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -90,6 +96,7 @@ public class TestLeafQueue {
RecordFactoryProvider.getRecordFactory(null);
RMContext rmContext;
+ RMContext spyRMContext;
CapacityScheduler cs;
CapacitySchedulerConfiguration csConf;
CapacitySchedulerContext csContext;
@@ -107,6 +114,14 @@ public class TestLeafQueue {
CapacityScheduler spyCs = new CapacityScheduler();
cs = spy(spyCs);
rmContext = TestUtils.getMockRMContext();
+ spyRMContext = spy(rmContext);
+
+ ConcurrentMap<ApplicationId, RMApp> spyApps =
+ spy(new ConcurrentHashMap<ApplicationId, RMApp>());
+ RMApp rmApp = mock(RMApp.class);
+ when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
+ Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
+ when(spyRMContext.getRMApps()).thenReturn(spyApps);
csConf =
new CapacitySchedulerConfiguration();
@@ -143,7 +158,7 @@ public class TestLeafQueue {
queues, queues,
TestUtils.spyHook);
- cs.setRMContext(rmContext);
+ cs.setRMContext(spyRMContext);
cs.init(csConf);
cs.start();
}
@@ -280,14 +295,14 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_1, user_0); // same user
@@ -329,14 +344,14 @@ public class TestLeafQueue {
final ApplicationAttemptId appAttemptId_0 = TestUtils
.getMockApplicationAttemptId(0, 1);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_d, d, null,
- rmContext);
+ spyRMContext);
d.submitApplicationAttempt(app_0, user_d);
// Attempt the same application again
final ApplicationAttemptId appAttemptId_1 = TestUtils
.getMockApplicationAttemptId(0, 2);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_d, d, null,
- rmContext);
+ spyRMContext);
d.submitApplicationAttempt(app_1, user_d); // same user
}
@@ -373,7 +388,7 @@ public class TestLeafQueue {
final ApplicationAttemptId appAttemptId_1 = TestUtils
.getMockApplicationAttemptId(0, 2);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, null,
- rmContext);
+ spyRMContext);
a.submitApplicationAttempt(app_1, user_0); // same user
assertEquals(1, a.getMetrics().getAppsSubmitted());
@@ -411,14 +426,14 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_1, user_0); // same user
@@ -545,21 +560,21 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- a.getActiveUsersManager(), rmContext);
+ a.getActiveUsersManager(), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- a.getActiveUsersManager(), rmContext);
+ a.getActiveUsersManager(), spyRMContext);
a.submitApplicationAttempt(app_1, user_0); // same user
final ApplicationAttemptId appAttemptId_2 =
TestUtils.getMockApplicationAttemptId(2, 0);
FiCaSchedulerApp app_2 =
new FiCaSchedulerApp(appAttemptId_2, user_1, a,
- a.getActiveUsersManager(), rmContext);
+ a.getActiveUsersManager(), spyRMContext);
a.submitApplicationAttempt(app_2, user_1);
// Setup some nodes
@@ -639,21 +654,21 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- a.getActiveUsersManager(), rmContext);
+ a.getActiveUsersManager(), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- a.getActiveUsersManager(), rmContext);
+ a.getActiveUsersManager(), spyRMContext);
a.submitApplicationAttempt(app_1, user_0); // same user
final ApplicationAttemptId appAttemptId_2 =
TestUtils.getMockApplicationAttemptId(2, 0);
FiCaSchedulerApp app_2 =
new FiCaSchedulerApp(appAttemptId_2, user_1, a,
- a.getActiveUsersManager(), rmContext);
+ a.getActiveUsersManager(), spyRMContext);
a.submitApplicationAttempt(app_2, user_1);
// Setup some nodes
@@ -750,28 +765,28 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- a.getActiveUsersManager(), rmContext);
+ a.getActiveUsersManager(), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- a.getActiveUsersManager(), rmContext);
+ a.getActiveUsersManager(), spyRMContext);
a.submitApplicationAttempt(app_1, user_0); // same user
final ApplicationAttemptId appAttemptId_2 =
TestUtils.getMockApplicationAttemptId(2, 0);
FiCaSchedulerApp app_2 =
new FiCaSchedulerApp(appAttemptId_2, user_1, a,
- a.getActiveUsersManager(), rmContext);
+ a.getActiveUsersManager(), spyRMContext);
a.submitApplicationAttempt(app_2, user_1);
final ApplicationAttemptId appAttemptId_3 =
TestUtils.getMockApplicationAttemptId(3, 0);
FiCaSchedulerApp app_3 =
new FiCaSchedulerApp(appAttemptId_3, user_2, a,
- a.getActiveUsersManager(), rmContext);
+ a.getActiveUsersManager(), spyRMContext);
a.submitApplicationAttempt(app_3, user_2);
// Setup some nodes
@@ -935,14 +950,14 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
new FiCaSchedulerApp(appAttemptId_1, user_1, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_1, user_1);
// Setup some nodes
@@ -1043,14 +1058,14 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
new FiCaSchedulerApp(appAttemptId_1, user_1, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_1, user_1);
// Setup some nodes
@@ -1150,14 +1165,14 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
new FiCaSchedulerApp(appAttemptId_1, user_1, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_1, user_1);
// Setup some nodes
@@ -1277,7 +1292,7 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
spy(new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext));
+ mock(ActiveUsersManager.class), spyRMContext));
a.submitApplicationAttempt(app_0, user_0);
// Setup some nodes and racks
@@ -1418,7 +1433,7 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
spy(new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext));
+ mock(ActiveUsersManager.class), spyRMContext));
a.submitApplicationAttempt(app_0, user_0);
// Setup some nodes and racks
@@ -1549,7 +1564,7 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
spy(new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext));
+ mock(ActiveUsersManager.class), spyRMContext));
a.submitApplicationAttempt(app_0, user_0);
// Setup some nodes and racks
@@ -1652,21 +1667,21 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
new FiCaSchedulerApp(appAttemptId_0, user_e, e,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
e.submitApplicationAttempt(app_0, user_e);
final ApplicationAttemptId appAttemptId_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
new FiCaSchedulerApp(appAttemptId_1, user_e, e,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
e.submitApplicationAttempt(app_1, user_e); // same user
final ApplicationAttemptId appAttemptId_2 =
TestUtils.getMockApplicationAttemptId(2, 0);
FiCaSchedulerApp app_2 =
new FiCaSchedulerApp(appAttemptId_2, user_e, e,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
e.submitApplicationAttempt(app_2, user_e); // same user
// before reinitialization
@@ -1730,21 +1745,21 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
new FiCaSchedulerApp(appAttemptId_0, user_e, e,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
e.submitApplicationAttempt(app_0, user_e);
final ApplicationAttemptId appAttemptId_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
new FiCaSchedulerApp(appAttemptId_1, user_e, e,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
e.submitApplicationAttempt(app_1, user_e); // same user
final ApplicationAttemptId appAttemptId_2 =
TestUtils.getMockApplicationAttemptId(2, 0);
FiCaSchedulerApp app_2 =
new FiCaSchedulerApp(appAttemptId_2, user_e, e,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
e.submitApplicationAttempt(app_2, user_e); // same user
// before updating cluster resource
@@ -1807,14 +1822,14 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
spy(new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext));
+ mock(ActiveUsersManager.class), spyRMContext));
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
spy(new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- mock(ActiveUsersManager.class), rmContext));
+ mock(ActiveUsersManager.class), spyRMContext));
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes and racks
@@ -2062,14 +2077,14 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_1, user_0); // same user
// Setup some nodes
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index 61def87..bd7f1bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -162,7 +162,7 @@ public class FairSchedulerTestBase {
RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt);
when(rmAppAttempt.getRMAppAttemptMetrics()).thenReturn(
- new RMAppAttemptMetrics(id));
+ new RMAppAttemptMetrics(id, resourceManager.getRMContext()));
resourceManager.getRMContext().getRMApps()
.put(id.getApplicationId(), rmApp);
return id;
@@ -183,7 +183,7 @@ public class FairSchedulerTestBase {
RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt);
when(rmAppAttempt.getRMAppAttemptMetrics()).thenReturn(
- new RMAppAttemptMetrics(id));
+ new RMAppAttemptMetrics(id,resourceManager.getRMContext()));
resourceManager.getRMContext().getRMApps()
.put(id.getApplicationId(), rmApp);
return id;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
index 111bf47..f07cb8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
@@ -136,7 +136,7 @@ public class TestRMWebAppFairScheduler {
MockRMApp app = new MockRMApp(i, i, state) {
@Override
public RMAppMetrics getRMAppMetrics() {
- return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0);
+ return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
}
@Override
public YarnApplicationState createApplicationState() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index b009bfa..3701dd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1322,7 +1322,7 @@ public class TestRMWebServicesApps extends JerseyTest {
Exception {
// 28 because trackingUrl not assigned yet
- assertEquals("incorrect number of elements", 24, info.length());
+ assertEquals("incorrect number of elements", 26, info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
info.getString("name"), info.getString("applicationType"),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
index 9609ba3..0fd5b24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
@@ -1197,7 +1197,9 @@ ResourceManager REST API's.
"queue" : "default",
"allocatedMB" : 0,
"allocatedVCores" : 0,
- "runningContainers" : 0
+ "runningContainers" : 0,
+ "memorySeconds" : 151730,
+ "vcoreSeconds" : 103
},
{
"finishedTime" : 1326815789546,
@@ -1218,7 +1220,9 @@ ResourceManager REST API's.
"queue" : "default",
"allocatedMB" : 0,
"allocatedVCores" : 0,
- "runningContainers" : 1
+ "runningContainers" : 1,
+ "memorySeconds" : 640064,
+ "vcoreSeconds" : 442
}
]
}
@@ -1271,6 +1275,8 @@ _01_000001</amContainerLogs>
<allocatedMB>0</allocatedMB>
<allocatedVCores>0</allocatedVCores>
<runningContainers>0</runningContainers>
+ <memorySeconds>151730</memorySeconds>
+ <vcoreSeconds>103</vcoreSeconds>
</app>
<app>
<id>application_1326815542473_0002</id>
@@ -1293,6 +1299,8 @@ _01_000001</amContainerLogs>
<allocatedMB>0</allocatedMB>
<allocatedVCores>0</allocatedVCores>
<runningContainers>0</runningContainers>
+ <memorySeconds>640064</memorySeconds>
+ <vcoreSeconds>442</vcoreSeconds>
</app>
</apps>
+---+
@@ -1491,6 +1499,10 @@ _01_000001</amContainerLogs>
+---------------------------------------------------------------+
| runningContainers | int | The number of containers currently running for the application |
+---------------------------------------------------------------+
+| memorySeconds | long | The amount of memory the application has allocated (megabyte-seconds) |
+*---------------+--------------+--------------------------------+
+| vcoreSeconds | long | The amount of CPU resources the application has allocated (virtual core-seconds) |
+*---------------+--------------+--------------------------------+
** Response Examples
@@ -1532,7 +1544,9 @@ _01_000001</amContainerLogs>
"elapsedTime" : 446748,
"diagnostics" : "",
"trackingUrl" : "http://host.domain.com:8088/proxy/application_1326821518301_0005/jobhistory/job/job_1326821518301_5_5",
- "queue" : "a1"
+ "queue" : "a1",
+ "memorySeconds" : 151730,
+ "vcoreSeconds" : 103
}
}
+---+
@@ -1576,6 +1590,8 @@ _01_000001</amContainerLogs>
<elapsedTime>446748</elapsedTime>
<amContainerLogs>http://host.domain.com:8042/node/containerlogs/container_1326821518301_0005_01_000001</amContainerLogs>
<amHostHttpAddress>host.domain.com:8042</amHostHttpAddress>
+ <memorySeconds>151730</memorySeconds>
+ <vcoreSeconds>103</vcoreSeconds>
</app>
+---+
[13/20] HADOOP-11074. Move s3-related FS connector code to
hadoop-aws. (David S. Wang via Colin Patrick McCabe)
Posted by sz...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3Credentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3Credentials.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3Credentials.java
new file mode 100644
index 0000000..312bf65
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3Credentials.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.net.URI;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * <p>
+ * Extracts AWS credentials from the filesystem URI or configuration.
+ * </p>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class S3Credentials {
+
+ private String accessKey;
+ private String secretAccessKey;
+
+ /**
+ * @throws IllegalArgumentException if credentials for S3 cannot be
+ * determined.
+ */
+ public void initialize(URI uri, Configuration conf) {
+ if (uri.getHost() == null) {
+ throw new IllegalArgumentException("Invalid hostname in URI " + uri);
+ }
+
+ String userInfo = uri.getUserInfo();
+ if (userInfo != null) {
+ int index = userInfo.indexOf(':');
+ if (index != -1) {
+ accessKey = userInfo.substring(0, index);
+ secretAccessKey = userInfo.substring(index + 1);
+ } else {
+ accessKey = userInfo;
+ }
+ }
+
+ String scheme = uri.getScheme();
+ String accessKeyProperty = String.format("fs.%s.awsAccessKeyId", scheme);
+ String secretAccessKeyProperty =
+ String.format("fs.%s.awsSecretAccessKey", scheme);
+ if (accessKey == null) {
+ accessKey = conf.get(accessKeyProperty);
+ }
+ if (secretAccessKey == null) {
+ secretAccessKey = conf.get(secretAccessKeyProperty);
+ }
+ if (accessKey == null && secretAccessKey == null) {
+ throw new IllegalArgumentException("AWS " +
+ "Access Key ID and Secret Access " +
+ "Key must be specified as the " +
+ "username or password " +
+ "(respectively) of a " + scheme +
+ " URL, or by setting the " +
+ accessKeyProperty + " or " +
+ secretAccessKeyProperty +
+ " properties (respectively).");
+ } else if (accessKey == null) {
+ throw new IllegalArgumentException("AWS " +
+ "Access Key ID must be specified " +
+ "as the username of a " + scheme +
+ " URL, or by setting the " +
+ accessKeyProperty + " property.");
+ } else if (secretAccessKey == null) {
+ throw new IllegalArgumentException("AWS " +
+ "Secret Access Key must be " +
+ "specified as the password of a " +
+ scheme + " URL, or by setting the " +
+ secretAccessKeyProperty +
+ " property.");
+ }
+
+ }
+
+ public String getAccessKey() {
+ return accessKey;
+ }
+
+ public String getSecretAccessKey() {
+ return secretAccessKey;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3Exception.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3Exception.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3Exception.java
new file mode 100644
index 0000000..4f07c4e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3Exception.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Thrown if there is a problem communicating with Amazon S3.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class S3Exception extends IOException {
+
+ private static final long serialVersionUID = 1L;
+
+ public S3Exception(Throwable t) {
+ super(t);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
new file mode 100644
index 0000000..dda3cf6
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
@@ -0,0 +1,486 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.s3native.NativeS3FileSystem;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryProxy;
+import org.apache.hadoop.util.Progressable;
+
+/**
+ * <p>
+ * A block-based {@link FileSystem} backed by
+ * <a href="http://aws.amazon.com/s3">Amazon S3</a>.
+ * </p>
+ * @see NativeS3FileSystem
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class S3FileSystem extends FileSystem {
+
+ private URI uri;
+
+ private FileSystemStore store;
+
+ private Path workingDir;
+
+ public S3FileSystem() {
+ // set store in initialize()
+ }
+
+ public S3FileSystem(FileSystemStore store) {
+ this.store = store;
+ }
+
+ /**
+ * Return the protocol scheme for the FileSystem.
+ * <p/>
+ *
+ * @return <code>s3</code>
+ */
+ @Override
+ public String getScheme() {
+ return "s3";
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public void initialize(URI uri, Configuration conf) throws IOException {
+ super.initialize(uri, conf);
+ if (store == null) {
+ store = createDefaultStore(conf);
+ }
+ store.initialize(uri, conf);
+ setConf(conf);
+ this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
+ this.workingDir =
+ new Path("/user", System.getProperty("user.name")).makeQualified(this);
+ }
+
+ private static FileSystemStore createDefaultStore(Configuration conf) {
+ FileSystemStore store = new Jets3tFileSystemStore();
+
+ RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
+ conf.getInt("fs.s3.maxRetries", 4),
+ conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
+ Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
+ new HashMap<Class<? extends Exception>, RetryPolicy>();
+ exceptionToPolicyMap.put(IOException.class, basePolicy);
+ exceptionToPolicyMap.put(S3Exception.class, basePolicy);
+
+ RetryPolicy methodPolicy = RetryPolicies.retryByException(
+ RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
+ Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
+ methodNameToPolicyMap.put("storeBlock", methodPolicy);
+ methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
+
+ return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
+ store, methodNameToPolicyMap);
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ @Override
+ public void setWorkingDirectory(Path dir) {
+ workingDir = makeAbsolute(dir);
+ }
+
+ private Path makeAbsolute(Path path) {
+ if (path.isAbsolute()) {
+ return path;
+ }
+ return new Path(workingDir, path);
+ }
+
+ /**
+ * @param permission Currently ignored.
+ */
+ @Override
+ public boolean mkdirs(Path path, FsPermission permission) throws IOException {
+ Path absolutePath = makeAbsolute(path);
+ List<Path> paths = new ArrayList<Path>();
+ do {
+ paths.add(0, absolutePath);
+ absolutePath = absolutePath.getParent();
+ } while (absolutePath != null);
+
+ boolean result = true;
+ for (Path p : paths) {
+ result &= mkdir(p);
+ }
+ return result;
+ }
+
+ private boolean mkdir(Path path) throws IOException {
+ Path absolutePath = makeAbsolute(path);
+ INode inode = store.retrieveINode(absolutePath);
+ if (inode == null) {
+ store.storeINode(absolutePath, INode.DIRECTORY_INODE);
+ } else if (inode.isFile()) {
+ throw new IOException(String.format(
+ "Can't make directory for path %s since it is a file.",
+ absolutePath));
+ }
+ return true;
+ }
+
+ @Override
+ public boolean isFile(Path path) throws IOException {
+ INode inode = store.retrieveINode(makeAbsolute(path));
+ if (inode == null) {
+ return false;
+ }
+ return inode.isFile();
+ }
+
+ private INode checkFile(Path path) throws IOException {
+ INode inode = store.retrieveINode(makeAbsolute(path));
+ if (inode == null) {
+ throw new IOException("No such file.");
+ }
+ if (inode.isDirectory()) {
+ throw new IOException("Path " + path + " is a directory.");
+ }
+ return inode;
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+ Path absolutePath = makeAbsolute(f);
+ INode inode = store.retrieveINode(absolutePath);
+ if (inode == null) {
+ throw new FileNotFoundException("File " + f + " does not exist.");
+ }
+ if (inode.isFile()) {
+ return new FileStatus[] {
+ new S3FileStatus(f.makeQualified(this), inode)
+ };
+ }
+ ArrayList<FileStatus> ret = new ArrayList<FileStatus>();
+ for (Path p : store.listSubPaths(absolutePath)) {
+ ret.add(getFileStatus(p.makeQualified(this)));
+ }
+ return ret.toArray(new FileStatus[0]);
+ }
+
+ /** This optional operation is not yet supported. */
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new IOException("Not supported");
+ }
+
+ /**
+ * @param permission Currently ignored.
+ */
+ @Override
+ public FSDataOutputStream create(Path file, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize, Progressable progress)
+ throws IOException {
+
+ INode inode = store.retrieveINode(makeAbsolute(file));
+ if (inode != null) {
+ if (overwrite) {
+ delete(file, true);
+ } else {
+ throw new FileAlreadyExistsException("File already exists: " + file);
+ }
+ } else {
+ Path parent = file.getParent();
+ if (parent != null) {
+ if (!mkdirs(parent)) {
+ throw new IOException("Mkdirs failed to create " + parent.toString());
+ }
+ }
+ }
+ return new FSDataOutputStream
+ (new S3OutputStream(getConf(), store, makeAbsolute(file),
+ blockSize, progress, bufferSize),
+ statistics);
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ INode inode = checkFile(path);
+ return new FSDataInputStream(new S3InputStream(getConf(), store, inode,
+ statistics));
+ }
+
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ Path absoluteSrc = makeAbsolute(src);
+ final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
+ INode srcINode = store.retrieveINode(absoluteSrc);
+ boolean debugEnabled = LOG.isDebugEnabled();
+ if (srcINode == null) {
+ // src path doesn't exist
+ if (debugEnabled) {
+ LOG.debug(debugPreamble + "returning false as src does not exist");
+ }
+ return false;
+ }
+
+ Path absoluteDst = makeAbsolute(dst);
+
+ //validate the parent dir of the destination
+ Path dstParent = absoluteDst.getParent();
+ if (dstParent != null) {
+ //if the dst parent is not root, make sure it exists
+ INode dstParentINode = store.retrieveINode(dstParent);
+ if (dstParentINode == null) {
+ // dst parent doesn't exist
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "returning false as dst parent does not exist");
+ }
+ return false;
+ }
+ if (dstParentINode.isFile()) {
+ // dst parent exists but is a file
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "returning false as dst parent exists and is a file");
+ }
+ return false;
+ }
+ }
+
+ //get status of source
+ boolean srcIsFile = srcINode.isFile();
+
+ INode dstINode = store.retrieveINode(absoluteDst);
+ boolean destExists = dstINode != null;
+ boolean destIsDir = destExists && !dstINode.isFile();
+ if (srcIsFile) {
+
+ //source is a simple file
+ if (destExists) {
+ if (destIsDir) {
+ //outcome #1 dest exists and is dir -filename to subdir of dest
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "copying src file under dest dir to " + absoluteDst);
+ }
+ absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
+ } else {
+ //outcome #2 dest it's a file: fail iff different from src
+ boolean renamingOnToSelf = absoluteSrc.equals(absoluteDst);
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "copying file onto file, outcome is " + renamingOnToSelf);
+ }
+ return renamingOnToSelf;
+ }
+ } else {
+ // #3 dest does not exist: use dest as path for rename
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "copying file onto file");
+ }
+ }
+ } else {
+ //here the source exists and is a directory
+ // outcomes (given we know the parent dir exists if we get this far)
+ // #1 destination is a file: fail
+ // #2 destination is a directory: create a new dir under that one
+ // #3 destination doesn't exist: create a new dir with that name
+ // #3 and #4 are only allowed if the dest path is not == or under src
+
+ if (destExists) {
+ if (!destIsDir) {
+ // #1 destination is a file: fail
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "returning false as src is a directory, but not dest");
+ }
+ return false;
+ } else {
+ // the destination dir exists
+ // destination for rename becomes a subdir of the target name
+ absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "copying src dir under dest dir to " + absoluteDst);
+ }
+ }
+ }
+ //the final destination directory is now know, so validate it for
+ //illegal moves
+
+ if (absoluteSrc.equals(absoluteDst)) {
+ //you can't rename a directory onto itself
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "Dest==source && isDir -failing");
+ }
+ return false;
+ }
+ if (absoluteDst.toString().startsWith(absoluteSrc.toString() + "/")) {
+ //you can't move a directory under itself
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "dst is equal to or under src dir -failing");
+ }
+ return false;
+ }
+ }
+ //here the dest path is set up -so rename
+ return renameRecursive(absoluteSrc, absoluteDst);
+ }
+
+ private boolean renameRecursive(Path src, Path dst) throws IOException {
+ INode srcINode = store.retrieveINode(src);
+ store.storeINode(dst, srcINode);
+ store.deleteINode(src);
+ if (srcINode.isDirectory()) {
+ for (Path oldSrc : store.listDeepSubPaths(src)) {
+ INode inode = store.retrieveINode(oldSrc);
+ if (inode == null) {
+ return false;
+ }
+ String oldSrcPath = oldSrc.toUri().getPath();
+ String srcPath = src.toUri().getPath();
+ String dstPath = dst.toUri().getPath();
+ Path newDst = new Path(oldSrcPath.replaceFirst(srcPath, dstPath));
+ store.storeINode(newDst, inode);
+ store.deleteINode(oldSrc);
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ Path absolutePath = makeAbsolute(path);
+ INode inode = store.retrieveINode(absolutePath);
+ if (inode == null) {
+ return false;
+ }
+ if (inode.isFile()) {
+ store.deleteINode(absolutePath);
+ for (Block block: inode.getBlocks()) {
+ store.deleteBlock(block);
+ }
+ } else {
+ FileStatus[] contents = null;
+ try {
+ contents = listStatus(absolutePath);
+ } catch(FileNotFoundException fnfe) {
+ return false;
+ }
+
+ if ((contents.length !=0) && (!recursive)) {
+ throw new IOException("Directory " + path.toString()
+ + " is not empty.");
+ }
+ for (FileStatus p:contents) {
+ if (!delete(p.getPath(), recursive)) {
+ return false;
+ }
+ }
+ store.deleteINode(absolutePath);
+ }
+ return true;
+ }
+
+ /**
+ * FileStatus for S3 file systems.
+ */
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ INode inode = store.retrieveINode(makeAbsolute(f));
+ if (inode == null) {
+ throw new FileNotFoundException(f + ": No such file or directory.");
+ }
+ return new S3FileStatus(f.makeQualified(this), inode);
+ }
+
+ @Override
+ public long getDefaultBlockSize() {
+ return getConf().getLong("fs.s3.block.size", 64 * 1024 * 1024);
+ }
+
+ @Override
+ public String getCanonicalServiceName() {
+ // Does not support Token
+ return null;
+ }
+
+ // diagnostic methods
+
+ void dump() throws IOException {
+ store.dump();
+ }
+
+ void purge() throws IOException {
+ store.purge();
+ }
+
+ private static class S3FileStatus extends FileStatus {
+
+ S3FileStatus(Path f, INode inode) throws IOException {
+ super(findLength(inode), inode.isDirectory(), 1,
+ findBlocksize(inode), 0, f);
+ }
+
+ private static long findLength(INode inode) {
+ if (!inode.isDirectory()) {
+ long length = 0L;
+ for (Block block : inode.getBlocks()) {
+ length += block.getLength();
+ }
+ return length;
+ }
+ return 0;
+ }
+
+ private static long findBlocksize(INode inode) {
+ final Block[] ret = inode.getBlocks();
+ return ret == null ? 0L : ret[0].getLength();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemConfigKeys.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemConfigKeys.java
new file mode 100644
index 0000000..8172a46
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemConfigKeys.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+
+/**
+ * This class contains constants for configuration keys used
+ * in the s3 file system.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class S3FileSystemConfigKeys extends CommonConfigurationKeys {
+ public static final String S3_BLOCK_SIZE_KEY = "s3.blocksize";
+ public static final long S3_BLOCK_SIZE_DEFAULT = 64*1024*1024;
+ public static final String S3_REPLICATION_KEY = "s3.replication";
+ public static final short S3_REPLICATION_DEFAULT = 1;
+ public static final String S3_STREAM_BUFFER_SIZE_KEY =
+ "s3.stream-buffer-size";
+ public static final int S3_STREAM_BUFFER_SIZE_DEFAULT = 4096;
+ public static final String S3_BYTES_PER_CHECKSUM_KEY =
+ "s3.bytes-per-checksum";
+ public static final int S3_BYTES_PER_CHECKSUM_DEFAULT = 512;
+ public static final String S3_CLIENT_WRITE_PACKET_SIZE_KEY =
+ "s3.client-write-packet-size";
+ public static final int S3_CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024;
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemException.java
new file mode 100644
index 0000000..cc1b463
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystemException.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Thrown when there is a fatal exception while using {@link S3FileSystem}.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class S3FileSystemException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ public S3FileSystemException(String message) {
+ super(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
new file mode 100644
index 0000000..5af57e6
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class S3InputStream extends FSInputStream {
+
+ private FileSystemStore store;
+
+ private Block[] blocks;
+
+ private boolean closed;
+
+ private long fileLength;
+
+ private long pos = 0;
+
+ private File blockFile;
+
+ private DataInputStream blockStream;
+
+ private long blockEnd = -1;
+
+ private FileSystem.Statistics stats;
+
+ private static final Log LOG =
+ LogFactory.getLog(S3InputStream.class.getName());
+
+
+ @Deprecated
+ public S3InputStream(Configuration conf, FileSystemStore store,
+ INode inode) {
+ this(conf, store, inode, null);
+ }
+
+ public S3InputStream(Configuration conf, FileSystemStore store,
+ INode inode, FileSystem.Statistics stats) {
+
+ this.store = store;
+ this.stats = stats;
+ this.blocks = inode.getBlocks();
+ for (Block block : blocks) {
+ this.fileLength += block.getLength();
+ }
+ }
+
+ @Override
+ public synchronized long getPos() throws IOException {
+ return pos;
+ }
+
+ @Override
+ public synchronized int available() throws IOException {
+ return (int) (fileLength - pos);
+ }
+
+ @Override
+ public synchronized void seek(long targetPos) throws IOException {
+ if (targetPos > fileLength) {
+ throw new IOException("Cannot seek after EOF");
+ }
+ pos = targetPos;
+ blockEnd = -1;
+ }
+
+ @Override
+ public synchronized boolean seekToNewSource(long targetPos) throws IOException {
+ return false;
+ }
+
+ @Override
+ public synchronized int read() throws IOException {
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+ int result = -1;
+ if (pos < fileLength) {
+ if (pos > blockEnd) {
+ blockSeekTo(pos);
+ }
+ result = blockStream.read();
+ if (result >= 0) {
+ pos++;
+ }
+ }
+ if (stats != null && result >= 0) {
+ stats.incrementBytesRead(1);
+ }
+ return result;
+ }
+
+ @Override
+ public synchronized int read(byte buf[], int off, int len) throws IOException {
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+ if (pos < fileLength) {
+ if (pos > blockEnd) {
+ blockSeekTo(pos);
+ }
+ int realLen = (int) Math.min((long) len, (blockEnd - pos + 1L));
+ int result = blockStream.read(buf, off, realLen);
+ if (result >= 0) {
+ pos += result;
+ }
+ if (stats != null && result > 0) {
+ stats.incrementBytesRead(result);
+ }
+ return result;
+ }
+ return -1;
+ }
+
+ private synchronized void blockSeekTo(long target) throws IOException {
+ //
+ // Compute desired block
+ //
+ int targetBlock = -1;
+ long targetBlockStart = 0;
+ long targetBlockEnd = 0;
+ for (int i = 0; i < blocks.length; i++) {
+ long blockLength = blocks[i].getLength();
+ targetBlockEnd = targetBlockStart + blockLength - 1;
+
+ if (target >= targetBlockStart && target <= targetBlockEnd) {
+ targetBlock = i;
+ break;
+ } else {
+ targetBlockStart = targetBlockEnd + 1;
+ }
+ }
+ if (targetBlock < 0) {
+ throw new IOException(
+ "Impossible situation: could not find target position " + target);
+ }
+ long offsetIntoBlock = target - targetBlockStart;
+
+ // read block blocks[targetBlock] from position offsetIntoBlock
+
+ this.blockFile = store.retrieveBlock(blocks[targetBlock], offsetIntoBlock);
+
+ this.pos = target;
+ this.blockEnd = targetBlockEnd;
+ this.blockStream = new DataInputStream(new FileInputStream(blockFile));
+
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ if (blockStream != null) {
+ blockStream.close();
+ blockStream = null;
+ }
+ if (blockFile != null) {
+ boolean b = blockFile.delete();
+ if (!b) {
+ LOG.warn("Ignoring failed delete");
+ }
+ }
+ super.close();
+ closed = true;
+ }
+
+ /**
+ * We don't support marks.
+ */
+ @Override
+ public boolean markSupported() {
+ return false;
+ }
+
+ @Override
+ public void mark(int readLimit) {
+ // Do nothing
+ }
+
+ @Override
+ public void reset() throws IOException {
+ throw new IOException("Mark not supported");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3OutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3OutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3OutputStream.java
new file mode 100644
index 0000000..761f2ce
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3OutputStream.java
@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3.INode.FileType;
+import org.apache.hadoop.util.Progressable;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class S3OutputStream extends OutputStream {
+
+ private Configuration conf;
+
+ private int bufferSize;
+
+ private FileSystemStore store;
+
+ private Path path;
+
+ private long blockSize;
+
+ private File backupFile;
+
+ private OutputStream backupStream;
+
+ private Random r = new Random();
+
+ private boolean closed;
+
+ private int pos = 0;
+
+ private long filePos = 0;
+
+ private int bytesWrittenToBlock = 0;
+
+ private byte[] outBuf;
+
+ private List<Block> blocks = new ArrayList<Block>();
+
+ private Block nextBlock;
+
+ private static final Log LOG =
+ LogFactory.getLog(S3OutputStream.class.getName());
+
+
+ public S3OutputStream(Configuration conf, FileSystemStore store,
+ Path path, long blockSize, Progressable progress,
+ int buffersize) throws IOException {
+
+ this.conf = conf;
+ this.store = store;
+ this.path = path;
+ this.blockSize = blockSize;
+ this.backupFile = newBackupFile();
+ this.backupStream = new FileOutputStream(backupFile);
+ this.bufferSize = buffersize;
+ this.outBuf = new byte[bufferSize];
+
+ }
+
+ private File newBackupFile() throws IOException {
+ File dir = new File(conf.get("fs.s3.buffer.dir"));
+ if (!dir.exists() && !dir.mkdirs()) {
+ throw new IOException("Cannot create S3 buffer directory: " + dir);
+ }
+ File result = File.createTempFile("output-", ".tmp", dir);
+ result.deleteOnExit();
+ return result;
+ }
+
+ public long getPos() throws IOException {
+ return filePos;
+ }
+
+ @Override
+ public synchronized void write(int b) throws IOException {
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+
+ if ((bytesWrittenToBlock + pos == blockSize) || (pos >= bufferSize)) {
+ flush();
+ }
+ outBuf[pos++] = (byte) b;
+ filePos++;
+ }
+
+ @Override
+ public synchronized void write(byte b[], int off, int len) throws IOException {
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+ while (len > 0) {
+ int remaining = bufferSize - pos;
+ int toWrite = Math.min(remaining, len);
+ System.arraycopy(b, off, outBuf, pos, toWrite);
+ pos += toWrite;
+ off += toWrite;
+ len -= toWrite;
+ filePos += toWrite;
+
+ if ((bytesWrittenToBlock + pos >= blockSize) || (pos == bufferSize)) {
+ flush();
+ }
+ }
+ }
+
+ @Override
+ public synchronized void flush() throws IOException {
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+
+ if (bytesWrittenToBlock + pos >= blockSize) {
+ flushData((int) blockSize - bytesWrittenToBlock);
+ }
+ if (bytesWrittenToBlock == blockSize) {
+ endBlock();
+ }
+ flushData(pos);
+ }
+
+ private synchronized void flushData(int maxPos) throws IOException {
+ int workingPos = Math.min(pos, maxPos);
+
+ if (workingPos > 0) {
+ //
+ // To the local block backup, write just the bytes
+ //
+ backupStream.write(outBuf, 0, workingPos);
+
+ //
+ // Track position
+ //
+ bytesWrittenToBlock += workingPos;
+ System.arraycopy(outBuf, workingPos, outBuf, 0, pos - workingPos);
+ pos -= workingPos;
+ }
+ }
+
+ private synchronized void endBlock() throws IOException {
+ //
+ // Done with local copy
+ //
+ backupStream.close();
+
+ //
+ // Send it to S3
+ //
+ // TODO: Use passed in Progressable to report progress.
+ nextBlockOutputStream();
+ store.storeBlock(nextBlock, backupFile);
+ internalClose();
+
+ //
+ // Delete local backup, start new one
+ //
+ boolean b = backupFile.delete();
+ if (!b) {
+ LOG.warn("Ignoring failed delete");
+ }
+ backupFile = newBackupFile();
+ backupStream = new FileOutputStream(backupFile);
+ bytesWrittenToBlock = 0;
+ }
+
+ private synchronized void nextBlockOutputStream() throws IOException {
+ long blockId = r.nextLong();
+ while (store.blockExists(blockId)) {
+ blockId = r.nextLong();
+ }
+ nextBlock = new Block(blockId, bytesWrittenToBlock);
+ blocks.add(nextBlock);
+ bytesWrittenToBlock = 0;
+ }
+
+ private synchronized void internalClose() throws IOException {
+ INode inode = new INode(FileType.FILE, blocks.toArray(new Block[blocks
+ .size()]));
+ store.storeINode(path, inode);
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (closed) {
+ return;
+ }
+
+ flush();
+ if (filePos == 0 || bytesWrittenToBlock != 0) {
+ endBlock();
+ }
+
+ backupStream.close();
+ boolean b = backupFile.delete();
+ if (!b) {
+ LOG.warn("Ignoring failed delete");
+ }
+
+ super.close();
+
+ closed = true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/VersionMismatchException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/VersionMismatchException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/VersionMismatchException.java
new file mode 100644
index 0000000..ccc8969
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/VersionMismatchException.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Thrown when Hadoop cannot read the version of the data stored
+ * in {@link S3FileSystem}.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class VersionMismatchException extends S3FileSystemException {
+ private static final long serialVersionUID = 1L;
+
+ public VersionMismatchException(String clientVersion, String dataVersion) {
+ super("Version mismatch: client expects version " + clientVersion +
+ ", but data has version " +
+ (dataVersion == null ? "[unversioned]" : dataVersion));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/package.html
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/package.html b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/package.html
new file mode 100644
index 0000000..dd601e1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/package.html
@@ -0,0 +1,55 @@
+<html>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<body>
+
+<p>A distributed, block-based implementation of {@link
+org.apache.hadoop.fs.FileSystem} that uses <a href="http://aws.amazon.com/s3">Amazon S3</a>
+as a backing store.</p>
+
+<p>
+Files are stored in S3 as blocks (represented by
+{@link org.apache.hadoop.fs.s3.Block}), which have an ID and a length.
+Block metadata is stored in S3 as a small record (represented by
+{@link org.apache.hadoop.fs.s3.INode}) using the URL-encoded
+path string as a key. Inodes record the file type (regular file or directory) and the list of blocks.
+This design makes it easy to seek to any given position in a file by reading the inode data to compute
+which block to access, then using S3's support for
+<a href="http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.2">HTTP Range</a> headers
+to start streaming from the correct position.
+Renames are also efficient since only the inode is moved (by a DELETE followed by a PUT since
+S3 does not support renames).
+</p>
+<p>
+For a single file <i>/dir1/file1</i> which takes two blocks of storage, the file structure in S3
+would be something like this:
+</p>
+<pre>
+/
+/dir1
+/dir1/file1
+block-6415776850131549260
+block-3026438247347758425
+</pre>
+<p>
+Inodes start with a leading <code>/</code>, while blocks are prefixed with <code>block-</code>.
+</p>
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/FileMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/FileMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/FileMetadata.java
new file mode 100644
index 0000000..2746af4
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/FileMetadata.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * <p>
+ * Holds basic metadata for a file stored in a {@link NativeFileSystemStore}.
+ * </p>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class FileMetadata {
+ private final String key;
+ private final long length;
+ private final long lastModified;
+
+ public FileMetadata(String key, long length, long lastModified) {
+ this.key = key;
+ this.length = length;
+ this.lastModified = lastModified;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public long getLength() {
+ return length;
+ }
+
+ public long getLastModified() {
+ return lastModified;
+ }
+
+ @Override
+ public String toString() {
+ return "FileMetadata[" + key + ", " + length + ", " + lastModified + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
new file mode 100644
index 0000000..a10d6f2
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
@@ -0,0 +1,483 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import static org.apache.hadoop.fs.s3native.NativeS3FileSystem.PATH_DELIMITER;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.s3.S3Credentials;
+import org.apache.hadoop.fs.s3.S3Exception;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.AccessControlException;
+import org.jets3t.service.S3Service;
+import org.jets3t.service.S3ServiceException;
+import org.jets3t.service.ServiceException;
+import org.jets3t.service.StorageObjectsChunk;
+import org.jets3t.service.impl.rest.HttpException;
+import org.jets3t.service.impl.rest.httpclient.RestS3Service;
+import org.jets3t.service.model.MultipartPart;
+import org.jets3t.service.model.MultipartUpload;
+import org.jets3t.service.model.S3Bucket;
+import org.jets3t.service.model.S3Object;
+import org.jets3t.service.model.StorageObject;
+import org.jets3t.service.security.AWSCredentials;
+import org.jets3t.service.utils.MultipartUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class Jets3tNativeFileSystemStore implements NativeFileSystemStore {
+
+ private S3Service s3Service;
+ private S3Bucket bucket;
+
+ private long multipartBlockSize;
+ private boolean multipartEnabled;
+ private long multipartCopyBlockSize;
+ static final long MAX_PART_SIZE = (long)5 * 1024 * 1024 * 1024;
+
+ private String serverSideEncryptionAlgorithm;
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(Jets3tNativeFileSystemStore.class);
+
+ @Override
+ public void initialize(URI uri, Configuration conf) throws IOException {
+ S3Credentials s3Credentials = new S3Credentials();
+ s3Credentials.initialize(uri, conf);
+ try {
+ AWSCredentials awsCredentials =
+ new AWSCredentials(s3Credentials.getAccessKey(),
+ s3Credentials.getSecretAccessKey());
+ this.s3Service = new RestS3Service(awsCredentials);
+ } catch (S3ServiceException e) {
+ handleException(e);
+ }
+ multipartEnabled =
+ conf.getBoolean("fs.s3n.multipart.uploads.enabled", false);
+ multipartBlockSize = Math.min(
+ conf.getLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024),
+ MAX_PART_SIZE);
+ multipartCopyBlockSize = Math.min(
+ conf.getLong("fs.s3n.multipart.copy.block.size", MAX_PART_SIZE),
+ MAX_PART_SIZE);
+ serverSideEncryptionAlgorithm = conf.get("fs.s3n.server-side-encryption-algorithm");
+
+ bucket = new S3Bucket(uri.getHost());
+ }
+
+ @Override
+ public void storeFile(String key, File file, byte[] md5Hash)
+ throws IOException {
+
+ if (multipartEnabled && file.length() >= multipartBlockSize) {
+ storeLargeFile(key, file, md5Hash);
+ return;
+ }
+
+ BufferedInputStream in = null;
+ try {
+ in = new BufferedInputStream(new FileInputStream(file));
+ S3Object object = new S3Object(key);
+ object.setDataInputStream(in);
+ object.setContentType("binary/octet-stream");
+ object.setContentLength(file.length());
+ object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
+ if (md5Hash != null) {
+ object.setMd5Hash(md5Hash);
+ }
+ s3Service.putObject(bucket, object);
+ } catch (ServiceException e) {
+ handleException(e, key);
+ } finally {
+ IOUtils.closeStream(in);
+ }
+ }
+
+ public void storeLargeFile(String key, File file, byte[] md5Hash)
+ throws IOException {
+ S3Object object = new S3Object(key);
+ object.setDataInputFile(file);
+ object.setContentType("binary/octet-stream");
+ object.setContentLength(file.length());
+ object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
+ if (md5Hash != null) {
+ object.setMd5Hash(md5Hash);
+ }
+
+ List<StorageObject> objectsToUploadAsMultipart =
+ new ArrayList<StorageObject>();
+ objectsToUploadAsMultipart.add(object);
+ MultipartUtils mpUtils = new MultipartUtils(multipartBlockSize);
+
+ try {
+ mpUtils.uploadObjects(bucket.getName(), s3Service,
+ objectsToUploadAsMultipart, null);
+ } catch (Exception e) {
+ handleException(e, key);
+ }
+ }
+
+ @Override
+ public void storeEmptyFile(String key) throws IOException {
+ try {
+ S3Object object = new S3Object(key);
+ object.setDataInputStream(new ByteArrayInputStream(new byte[0]));
+ object.setContentType("binary/octet-stream");
+ object.setContentLength(0);
+ object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
+ s3Service.putObject(bucket, object);
+ } catch (ServiceException e) {
+ handleException(e, key);
+ }
+ }
+
+ @Override
+ public FileMetadata retrieveMetadata(String key) throws IOException {
+ StorageObject object = null;
+ try {
+ LOG.debug("Getting metadata for key: {} from bucket: {}",
+ key, bucket.getName());
+ object = s3Service.getObjectDetails(bucket.getName(), key);
+ return new FileMetadata(key, object.getContentLength(),
+ object.getLastModifiedDate().getTime());
+
+ } catch (ServiceException e) {
+ try {
+ // process
+ handleException(e, key);
+ return null;
+ } catch (FileNotFoundException fnfe) {
+ // and downgrade missing files
+ return null;
+ }
+ } finally {
+ if (object != null) {
+ object.closeDataInputStream();
+ }
+ }
+ }
+
+ /**
+ * @param key
+ * The key is the object name that is being retrieved from the S3 bucket
+ * @return
+ * This method returns null if the key is not found
+ * @throws IOException
+ */
+
+ @Override
+ public InputStream retrieve(String key) throws IOException {
+ try {
+ LOG.debug("Getting key: {} from bucket: {}",
+ key, bucket.getName());
+ S3Object object = s3Service.getObject(bucket.getName(), key);
+ return object.getDataInputStream();
+ } catch (ServiceException e) {
+ handleException(e, key);
+ return null; //return null if key not found
+ }
+ }
+
+ /**
+ *
+ * @param key
+ * The key is the object name that is being retrieved from the S3 bucket
+ * @return
+ * This method returns null if the key is not found
+ * @throws IOException
+ */
+
+ @Override
+ public InputStream retrieve(String key, long byteRangeStart)
+ throws IOException {
+ try {
+ LOG.debug("Getting key: {} from bucket: {} with byteRangeStart: {}",
+ key, bucket.getName(), byteRangeStart);
+ S3Object object = s3Service.getObject(bucket, key, null, null, null,
+ null, byteRangeStart, null);
+ return object.getDataInputStream();
+ } catch (ServiceException e) {
+ handleException(e, key);
+ return null;
+ }
+ }
+
+ @Override
+ public PartialListing list(String prefix, int maxListingLength)
+ throws IOException {
+ return list(prefix, maxListingLength, null, false);
+ }
+
+ @Override
+ public PartialListing list(String prefix, int maxListingLength, String priorLastKey,
+ boolean recurse) throws IOException {
+
+ return list(prefix, recurse ? null : PATH_DELIMITER, maxListingLength, priorLastKey);
+ }
+
+ /**
+ * list objects
+ * @param prefix prefix
+ * @param delimiter delimiter
+ * @param maxListingLength max no. of entries
+ * @param priorLastKey last key in any previous search
+ * @return a list of matches
+ * @throws IOException on any reported failure
+ */
+
+ private PartialListing list(String prefix, String delimiter,
+ int maxListingLength, String priorLastKey) throws IOException {
+ try {
+ if (!prefix.isEmpty() && !prefix.endsWith(PATH_DELIMITER)) {
+ prefix += PATH_DELIMITER;
+ }
+ StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
+ prefix, delimiter, maxListingLength, priorLastKey);
+
+ FileMetadata[] fileMetadata =
+ new FileMetadata[chunk.getObjects().length];
+ for (int i = 0; i < fileMetadata.length; i++) {
+ StorageObject object = chunk.getObjects()[i];
+ fileMetadata[i] = new FileMetadata(object.getKey(),
+ object.getContentLength(), object.getLastModifiedDate().getTime());
+ }
+ return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
+ chunk.getCommonPrefixes());
+ } catch (ServiceException e) {
+ handleException(e, prefix);
+ return null; // never returned - keep compiler happy
+ }
+ }
+
+ @Override
+ public void delete(String key) throws IOException {
+ try {
+ LOG.debug("Deleting key: {} from bucket: {}",
+ key, bucket.getName());
+ s3Service.deleteObject(bucket, key);
+ } catch (ServiceException e) {
+ handleException(e, key);
+ }
+ }
+
+ public void rename(String srcKey, String dstKey) throws IOException {
+ try {
+ s3Service.renameObject(bucket.getName(), srcKey, new S3Object(dstKey));
+ } catch (ServiceException e) {
+ handleException(e, srcKey);
+ }
+ }
+
+ @Override
+ public void copy(String srcKey, String dstKey) throws IOException {
+ try {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Copying srcKey: " + srcKey + "to dstKey: " + dstKey + "in bucket: " + bucket.getName());
+ }
+ if (multipartEnabled) {
+ S3Object object = s3Service.getObjectDetails(bucket, srcKey, null,
+ null, null, null);
+ if (multipartCopyBlockSize > 0 &&
+ object.getContentLength() > multipartCopyBlockSize) {
+ copyLargeFile(object, dstKey);
+ return;
+ }
+ }
+
+ S3Object dstObject = new S3Object(dstKey);
+ dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
+ s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(),
+ dstObject, false);
+ } catch (ServiceException e) {
+ handleException(e, srcKey);
+ }
+ }
+
+ public void copyLargeFile(S3Object srcObject, String dstKey) throws IOException {
+ try {
+ long partCount = srcObject.getContentLength() / multipartCopyBlockSize +
+ (srcObject.getContentLength() % multipartCopyBlockSize > 0 ? 1 : 0);
+
+ MultipartUpload multipartUpload = s3Service.multipartStartUpload
+ (bucket.getName(), dstKey, srcObject.getMetadataMap());
+
+ List<MultipartPart> listedParts = new ArrayList<MultipartPart>();
+ for (int i = 0; i < partCount; i++) {
+ long byteRangeStart = i * multipartCopyBlockSize;
+ long byteLength;
+ if (i < partCount - 1) {
+ byteLength = multipartCopyBlockSize;
+ } else {
+ byteLength = srcObject.getContentLength() % multipartCopyBlockSize;
+ if (byteLength == 0) {
+ byteLength = multipartCopyBlockSize;
+ }
+ }
+
+ MultipartPart copiedPart = s3Service.multipartUploadPartCopy
+ (multipartUpload, i + 1, bucket.getName(), srcObject.getKey(),
+ null, null, null, null, byteRangeStart,
+ byteRangeStart + byteLength - 1, null);
+ listedParts.add(copiedPart);
+ }
+
+ Collections.reverse(listedParts);
+ s3Service.multipartCompleteUpload(multipartUpload, listedParts);
+ } catch (ServiceException e) {
+ handleException(e, srcObject.getKey());
+ }
+ }
+
+ @Override
+ public void purge(String prefix) throws IOException {
+ String key = "";
+ try {
+ S3Object[] objects =
+ s3Service.listObjects(bucket.getName(), prefix, null);
+ for (S3Object object : objects) {
+ key = object.getKey();
+ s3Service.deleteObject(bucket, key);
+ }
+ } catch (S3ServiceException e) {
+ handleException(e, key);
+ }
+ }
+
+ @Override
+ public void dump() throws IOException {
+ StringBuilder sb = new StringBuilder("S3 Native Filesystem, ");
+ sb.append(bucket.getName()).append("\n");
+ try {
+ S3Object[] objects = s3Service.listObjects(bucket.getName());
+ for (S3Object object : objects) {
+ sb.append(object.getKey()).append("\n");
+ }
+ } catch (S3ServiceException e) {
+ handleException(e);
+ }
+ System.out.println(sb);
+ }
+
+ /**
+ * Handle any service exception by translating it into an IOException
+ * @param e exception
+ * @throws IOException exception -always
+ */
+ private void handleException(Exception e) throws IOException {
+ throw processException(e, e, "");
+ }
+ /**
+ * Handle any service exception by translating it into an IOException
+ * @param e exception
+ * @param key key sought from object store
+
+ * @throws IOException exception -always
+ */
+ private void handleException(Exception e, String key) throws IOException {
+ throw processException(e, e, key);
+ }
+
+ /**
+ * Handle any service exception by translating it into an IOException
+ * @param thrown exception
+ * @param original original exception -thrown if no other translation could
+ * be made
+ * @param key key sought from object store or "" for undefined
+ * @return an exception to throw. If isProcessingCause==true this may be null.
+ */
+ private IOException processException(Throwable thrown, Throwable original,
+ String key) {
+ IOException result;
+ if (thrown.getCause() != null) {
+ // recurse down
+ result = processException(thrown.getCause(), original, key);
+ } else if (thrown instanceof HttpException) {
+ // nested HttpException - examine error code and react
+ HttpException httpException = (HttpException) thrown;
+ String responseMessage = httpException.getResponseMessage();
+ int responseCode = httpException.getResponseCode();
+ String bucketName = "s3n://" + bucket.getName();
+ String text = String.format("%s : %03d : %s",
+ bucketName,
+ responseCode,
+ responseMessage);
+ String filename = !key.isEmpty() ? (bucketName + "/" + key) : text;
+ IOException ioe;
+ switch (responseCode) {
+ case 404:
+ result = new FileNotFoundException(filename);
+ break;
+ case 416: // invalid range
+ result = new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF
+ +": " + filename);
+ break;
+ case 403: //forbidden
+ result = new AccessControlException("Permission denied"
+ +": " + filename);
+ break;
+ default:
+ result = new IOException(text);
+ }
+ result.initCause(thrown);
+ } else if (thrown instanceof S3ServiceException) {
+ S3ServiceException se = (S3ServiceException) thrown;
+ LOG.debug(
+ "S3ServiceException: {}: {} : {}",
+ se.getS3ErrorCode(), se.getS3ErrorMessage(), se, se);
+ if ("InvalidRange".equals(se.getS3ErrorCode())) {
+ result = new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
+ } else {
+ result = new S3Exception(se);
+ }
+ } else if (thrown instanceof ServiceException) {
+ ServiceException se = (ServiceException) thrown;
+ LOG.debug("S3ServiceException: {}: {} : {}",
+ se.getErrorCode(), se.toString(), se, se);
+ result = new S3Exception(se);
+ } else if (thrown instanceof IOException) {
+ result = (IOException) thrown;
+ } else {
+ // here there is no exception derived yet.
+ // this means no inner cause, and no translation made yet.
+ // convert the original to an IOException -rather than just the
+ // exception at the base of the tree
+ result = new S3Exception(original);
+ }
+
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
new file mode 100644
index 0000000..f26cdac
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * <p>
+ * An abstraction for a key-based {@link File} store.
+ * </p>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+interface NativeFileSystemStore {
+
+ void initialize(URI uri, Configuration conf) throws IOException;
+
+ void storeFile(String key, File file, byte[] md5Hash) throws IOException;
+ void storeEmptyFile(String key) throws IOException;
+
+ FileMetadata retrieveMetadata(String key) throws IOException;
+ InputStream retrieve(String key) throws IOException;
+ InputStream retrieve(String key, long byteRangeStart) throws IOException;
+
+ PartialListing list(String prefix, int maxListingLength) throws IOException;
+ PartialListing list(String prefix, int maxListingLength, String priorLastKey, boolean recursive)
+ throws IOException;
+
+ void delete(String key) throws IOException;
+
+ void copy(String srcKey, String dstKey) throws IOException;
+
+ /**
+ * Delete all keys with the given prefix. Used for testing.
+ * @throws IOException
+ */
+ void purge(String prefix) throws IOException;
+
+ /**
+ * Diagnostic method to dump state to the console.
+ * @throws IOException
+ */
+ void dump() throws IOException;
+}
[08/20] git commit: HADOOP-10758. KMS: add ACLs on per key basis.
(tucu)
Posted by sz...@apache.org.
HADOOP-10758. KMS: add ACLs on per key basis. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b02a4b40
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b02a4b40
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b02a4b40
Branch: refs/heads/HDFS-6584
Commit: b02a4b40610e93eef6559db09a11d287e859446d
Parents: cbfe263
Author: Alejandro Abdelnur <tu...@apache.org>
Authored: Wed Sep 10 14:26:15 2014 -0700
Committer: Alejandro Abdelnur <tu...@apache.org>
Committed: Wed Sep 10 14:26:15 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../hadoop-kms/src/main/conf/kms-acls.xml | 38 +++
.../hadoop/crypto/key/kms/server/KMSACLs.java | 97 ++++++-
.../crypto/key/kms/server/KMSConfiguration.java | 9 +
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 17 +-
.../kms/server/KeyAuthorizationKeyProvider.java | 276 +++++++++++++++++++
.../hadoop-kms/src/site/apt/index.apt.vm | 106 +++++++
.../hadoop/crypto/key/kms/server/TestKMS.java | 236 +++++++++++++++-
.../server/TestKeyAuthorizationKeyProvider.java | 218 +++++++++++++++
9 files changed, 986 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b02a4b40/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index b2157d6..3cea14a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -509,6 +509,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11057. checknative command to probe for winutils.exe on windows.
(Xiaoyu Yao via cnauroth)
+ HADOOP-10758. KMS: add ACLs on per key basis. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b02a4b40/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
index cdff629..24a46b8 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
@@ -94,4 +94,42 @@
ACL for decrypt EncryptedKey CryptoExtension operations
</description>
</property>
+
+ <property>
+ <name>default.key.acl.MANAGEMENT</name>
+ <value>*</value>
+ <description>
+ default ACL for MANAGEMENT operations for all key acls that are not
+ explicitly defined.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.GENERATE_EEK</name>
+ <value>*</value>
+ <description>
+ default ACL for GENERATE_EEK operations for all key acls that are not
+ explicitly defined.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.DECRYPT_EEK</name>
+ <value>*</value>
+ <description>
+ default ACL for DECRYPT_EEK operations for all key acls that are not
+ explicitly defined.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.READ</name>
+ <value>*</value>
+ <description>
+ default ACL for READ operations for all key acls that are not
+ explicitly defined.
+ </description>
+ </property>
+
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b02a4b40/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
index 8a10bb2..530fe11 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.crypto.key.kms.server;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.kms.server.KMS.KMSOp;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyACLs;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyOpType;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -32,6 +34,7 @@ import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
/**
* Provides access to the <code>AccessControlList</code>s used by KMS,
@@ -39,7 +42,7 @@ import java.util.concurrent.TimeUnit;
* are defined has been updated.
*/
@InterfaceAudience.Private
-public class KMSACLs implements Runnable {
+public class KMSACLs implements Runnable, KeyACLs {
private static final Logger LOG = LoggerFactory.getLogger(KMSACLs.class);
private static final String UNAUTHORIZED_MSG_WITH_KEY =
@@ -67,6 +70,9 @@ public class KMSACLs implements Runnable {
private volatile Map<Type, AccessControlList> acls;
private volatile Map<Type, AccessControlList> blacklistedAcls;
+ private volatile Map<String, HashMap<KeyOpType, AccessControlList>> keyAcls;
+ private final Map<KeyOpType, AccessControlList> defaultKeyAcls =
+ new HashMap<KeyOpType, AccessControlList>();
private ScheduledExecutorService executorService;
private long lastReload;
@@ -74,14 +80,15 @@ public class KMSACLs implements Runnable {
if (conf == null) {
conf = loadACLs();
}
- setACLs(conf);
+ setKMSACLs(conf);
+ setKeyACLs(conf);
}
public KMSACLs() {
this(null);
}
- private void setACLs(Configuration conf) {
+ private void setKMSACLs(Configuration conf) {
Map<Type, AccessControlList> tempAcls = new HashMap<Type, AccessControlList>();
Map<Type, AccessControlList> tempBlacklist = new HashMap<Type, AccessControlList>();
for (Type aclType : Type.values()) {
@@ -99,14 +106,69 @@ public class KMSACLs implements Runnable {
blacklistedAcls = tempBlacklist;
}
+ private void setKeyACLs(Configuration conf) {
+ Map<String, HashMap<KeyOpType, AccessControlList>> tempKeyAcls =
+ new HashMap<String, HashMap<KeyOpType,AccessControlList>>();
+ Map<String, String> allKeyACLS =
+ conf.getValByRegex(Pattern.quote(KMSConfiguration.KEY_ACL_PREFIX));
+ for (Map.Entry<String, String> keyAcl : allKeyACLS.entrySet()) {
+ String k = keyAcl.getKey();
+ // this should be of type "key.acl.<KEY_NAME>.<OP_TYPE>"
+ int keyNameStarts = KMSConfiguration.KEY_ACL_PREFIX.length();
+ int keyNameEnds = k.lastIndexOf(".");
+ if (keyNameStarts >= keyNameEnds) {
+ LOG.warn("Invalid key name '{}'", k);
+ } else {
+ String aclStr = keyAcl.getValue();
+ String keyName = k.substring(keyNameStarts, keyNameEnds);
+ String keyOp = k.substring(keyNameEnds + 1);
+ KeyOpType aclType = null;
+ try {
+ aclType = KeyOpType.valueOf(keyOp);
+ } catch (IllegalArgumentException e) {
+ LOG.warn("Invalid key Operation '{}'", keyOp);
+ }
+ if (aclType != null) {
+ // On the assumption this will be single threaded.. else we need to
+ // ConcurrentHashMap
+ HashMap<KeyOpType,AccessControlList> aclMap =
+ tempKeyAcls.get(keyName);
+ if (aclMap == null) {
+ aclMap = new HashMap<KeyOpType, AccessControlList>();
+ tempKeyAcls.put(keyName, aclMap);
+ }
+ aclMap.put(aclType, new AccessControlList(aclStr));
+ LOG.info("KEY_NAME '{}' KEY_OP '{}' ACL '{}'",
+ keyName, aclType, aclStr);
+ }
+ }
+ }
+
+ keyAcls = tempKeyAcls;
+ for (KeyOpType keyOp : KeyOpType.values()) {
+ if (!defaultKeyAcls.containsKey(keyOp)) {
+ String confKey = KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + keyOp;
+ String aclStr = conf.get(confKey);
+ if (aclStr != null) {
+ if (aclStr.equals("*")) {
+ LOG.info("Default Key ACL for KEY_OP '{}' is set to '*'", keyOp);
+ }
+ defaultKeyAcls.put(keyOp, new AccessControlList(aclStr));
+ }
+ }
+ }
+ }
+
@Override
public void run() {
try {
if (KMSConfiguration.isACLsFileNewer(lastReload)) {
- setACLs(loadACLs());
+ setKMSACLs(loadACLs());
+ setKeyACLs(loadACLs());
}
} catch (Exception ex) {
- LOG.warn("Could not reload ACLs file: " + ex.toString(), ex);
+ LOG.warn(
+ String.format("Could not reload ACLs file: '%s'", ex.toString()), ex);
}
}
@@ -164,4 +226,29 @@ public class KMSACLs implements Runnable {
}
}
+ @Override
+ public boolean hasAccessToKey(String keyName, UserGroupInformation ugi,
+ KeyOpType opType) {
+ Map<KeyOpType, AccessControlList> keyAcl = keyAcls.get(keyName);
+ if (keyAcl == null) {
+ // Get KeyAcl map of DEFAULT KEY.
+ keyAcl = defaultKeyAcls;
+ }
+ // If No key acl defined for this key, check to see if
+ // there are key defaults configured for this operation
+ AccessControlList acl = keyAcl.get(opType);
+ if (acl == null) {
+ // If no acl is specified for this operation,
+ // deny access
+ return false;
+ } else {
+ return acl.isUserAllowed(ugi);
+ }
+ }
+
+ @Override
+ public boolean isACLPresent(String keyName, KeyOpType opType) {
+ return (keyAcls.containsKey(keyName) || defaultKeyAcls.containsKey(opType));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b02a4b40/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
index 35dccfc..a7daa24 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -36,6 +36,9 @@ public class KMSConfiguration {
public static final String CONFIG_PREFIX = "hadoop.kms.";
+ public static final String KEY_ACL_PREFIX = "key.acl.";
+ public static final String DEFAULT_KEY_ACL_PREFIX = "default.key.acl.";
+
// Property to Enable/Disable Caching
public static final String KEY_CACHE_ENABLE = CONFIG_PREFIX +
"cache.enable";
@@ -57,6 +60,12 @@ public class KMSConfiguration {
// 10 secs
public static final long KMS_AUDIT_AGGREGATION_DELAY_DEFAULT = 10000;
+ // Property to Enable/Disable per Key authorization
+ public static final String KEY_AUTHORIZATION_ENABLE = CONFIG_PREFIX +
+ "key.authorization.enable";
+
+ public static final boolean KEY_AUTHORIZATION_ENABLE_DEFAULT = true;
+
static Configuration getConfiguration(boolean loadHadoopDefaults,
String ... resources) {
Configuration conf = new Configuration(loadHadoopDefaults);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b02a4b40/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
index aaf90e8..0827b78 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
@@ -68,7 +68,7 @@ public class KMSWebApp implements ServletContextListener {
private JmxReporter jmxReporter;
private static Configuration kmsConf;
- private static KMSACLs acls;
+ private static KMSACLs kmsAcls;
private static Meter adminCallsMeter;
private static Meter keyCallsMeter;
private static Meter unauthorizedCallsMeter;
@@ -126,8 +126,8 @@ public class KMSWebApp implements ServletContextListener {
LOG.info(" KMS Hadoop Version: " + VersionInfo.getVersion());
LOG.info("-------------------------------------------------------------");
- acls = new KMSACLs();
- acls.startReloader();
+ kmsAcls = new KMSACLs();
+ kmsAcls.startReloader();
metricRegistry = new MetricRegistry();
jmxReporter = JmxReporter.forRegistry(metricRegistry).build();
@@ -188,6 +188,13 @@ public class KMSWebApp implements ServletContextListener {
keyProviderCryptoExtension =
new EagerKeyGeneratorKeyProviderCryptoExtension(kmsConf,
keyProviderCryptoExtension);
+ if (kmsConf.getBoolean(KMSConfiguration.KEY_AUTHORIZATION_ENABLE,
+ KMSConfiguration.KEY_AUTHORIZATION_ENABLE_DEFAULT)) {
+ keyProviderCryptoExtension =
+ new KeyAuthorizationKeyProvider(
+ keyProviderCryptoExtension, kmsAcls);
+ }
+
LOG.info("Initialized KeyProviderCryptoExtension "
+ keyProviderCryptoExtension);
final int defaultBitlength = kmsConf
@@ -213,7 +220,7 @@ public class KMSWebApp implements ServletContextListener {
@Override
public void contextDestroyed(ServletContextEvent sce) {
kmsAudit.shutdown();
- acls.stopReloader();
+ kmsAcls.stopReloader();
jmxReporter.stop();
jmxReporter.close();
metricRegistry = null;
@@ -225,7 +232,7 @@ public class KMSWebApp implements ServletContextListener {
}
public static KMSACLs getACLs() {
- return acls;
+ return kmsAcls;
}
public static Meter getAdminCallsMeter() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b02a4b40/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
new file mode 100644
index 0000000..fe908e3
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
@@ -0,0 +1,276 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.crypto.key.kms.server;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * A {@link KeyProvider} proxy that checks whether the current user derived via
+ * {@link UserGroupInformation}, is authorized to perform the following
+ * type of operations on a Key :
+ * <ol>
+ * <li>MANAGEMENT operations : createKey, rollNewVersion, deleteKey</li>
+ * <li>GENERATE_EEK operations : generateEncryptedKey, warmUpEncryptedKeys</li>
+ * <li>DECRYPT_EEK operation : decryptEncryptedKey</li>
+ * <li>READ operations : getKeyVersion, getKeyVersions, getMetadata,
+ * getKeysMetadata, getCurrentKey</li>
+ * </ol>
+ * The read operations (getCurrentKeyVersion / getMetadata) etc are not checked.
+ */
+public class KeyAuthorizationKeyProvider extends KeyProviderCryptoExtension {
+
+ public static final String KEY_ACL = "key.acl.";
+ private static final String KEY_ACL_NAME = KEY_ACL + "name";
+
+ public enum KeyOpType {
+ ALL, READ, MANAGEMENT, GENERATE_EEK, DECRYPT_EEK;
+ }
+
+ /**
+ * Interface that needs to be implemented by a client of the
+ * <code>KeyAuthorizationKeyProvider</code>.
+ */
+ public static interface KeyACLs {
+
+ /**
+ * This is called by the KeyProvider to check if the given user is
+ * authorized to perform the specified operation on the given acl name.
+ * @param aclName name of the key ACL
+ * @param ugi User's UserGroupInformation
+ * @param opType Operation Type
+ * @return true if user has access to the aclName and opType else false
+ */
+ public boolean hasAccessToKey(String aclName, UserGroupInformation ugi,
+ KeyOpType opType);
+
+ /**
+ *
+ * @param aclName ACL name
+ * @param opType Operation Type
+ * @return true if AclName exists else false
+ */
+ public boolean isACLPresent(String aclName, KeyOpType opType);
+ }
+
+ private final KeyProviderCryptoExtension provider;
+ private final KeyACLs acls;
+
+ /**
+ * The constructor takes a {@link KeyProviderCryptoExtension} and an
+ * implementation of <code>KeyACLs</code>. All calls are delegated to the
+ * provider keyProvider after authorization check (if required)
+ * @param keyProvider
+ * @param acls
+ */
+ public KeyAuthorizationKeyProvider(KeyProviderCryptoExtension keyProvider,
+ KeyACLs acls) {
+ super(keyProvider, null);
+ this.provider = keyProvider;
+ this.acls = acls;
+ }
+
+ // This method first checks if "key.acl.name" attribute is present as an
+ // attribute in the provider Options. If yes, use the aclName for any
+ // subsequent access checks, else use the keyName as the aclName and set it
+ // as the value of the "key.acl.name" in the key's metadata.
+ private void authorizeCreateKey(String keyName, Options options,
+ UserGroupInformation ugi) throws IOException{
+ Preconditions.checkNotNull(ugi, "UserGroupInformation cannot be null");
+ Map<String, String> attributes = options.getAttributes();
+ String aclName = attributes.get(KEY_ACL_NAME);
+ boolean success = false;
+ if (Strings.isNullOrEmpty(aclName)) {
+ if (acls.isACLPresent(keyName, KeyOpType.MANAGEMENT)) {
+ options.setAttributes(ImmutableMap.<String, String> builder()
+ .putAll(attributes).put(KEY_ACL_NAME, keyName).build());
+ success =
+ acls.hasAccessToKey(keyName, ugi, KeyOpType.MANAGEMENT)
+ || acls.hasAccessToKey(keyName, ugi, KeyOpType.ALL);
+ } else {
+ success = false;
+ }
+ } else {
+ success = acls.isACLPresent(aclName, KeyOpType.MANAGEMENT) &&
+ (acls.hasAccessToKey(aclName, ugi, KeyOpType.MANAGEMENT)
+ || acls.hasAccessToKey(aclName, ugi, KeyOpType.ALL));
+ }
+ if (!success)
+ throw new AuthorizationException(String.format("User [%s] is not"
+ + " authorized to create key !!", ugi.getShortUserName()));
+ }
+
+ private void checkAccess(String aclName, UserGroupInformation ugi,
+ KeyOpType opType) throws AuthorizationException {
+ Preconditions.checkNotNull(aclName, "Key ACL name cannot be null");
+ Preconditions.checkNotNull(ugi, "UserGroupInformation cannot be null");
+ if (acls.isACLPresent(aclName, KeyOpType.MANAGEMENT) &&
+ (acls.hasAccessToKey(aclName, ugi, opType)
+ || acls.hasAccessToKey(aclName, ugi, KeyOpType.ALL))) {
+ return;
+ } else {
+ throw new AuthorizationException(String.format("User [%s] is not"
+ + " authorized to perform [%s] on key with ACL name [%s]!!",
+ ugi.getShortUserName(), opType, aclName));
+ }
+ }
+
+ @Override
+ public KeyVersion createKey(String name, Options options)
+ throws NoSuchAlgorithmException, IOException {
+ authorizeCreateKey(name, options, getUser());
+ return provider.createKey(name, options);
+ }
+
+ @Override
+ public KeyVersion createKey(String name, byte[] material, Options options)
+ throws IOException {
+ authorizeCreateKey(name, options, getUser());
+ return provider.createKey(name, material, options);
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(String name)
+ throws NoSuchAlgorithmException, IOException {
+ doAccessCheck(name, KeyOpType.MANAGEMENT);
+ return provider.rollNewVersion(name);
+ }
+
+ @Override
+ public void deleteKey(String name) throws IOException {
+ doAccessCheck(name, KeyOpType.MANAGEMENT);
+ provider.deleteKey(name);
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(String name, byte[] material)
+ throws IOException {
+ doAccessCheck(name, KeyOpType.MANAGEMENT);
+ return provider.rollNewVersion(name, material);
+ }
+
+ @Override
+ public void warmUpEncryptedKeys(String... names) throws IOException {
+ for (String name : names) {
+ doAccessCheck(name, KeyOpType.GENERATE_EEK);
+ }
+ provider.warmUpEncryptedKeys(names);
+ }
+
+ @Override
+ public EncryptedKeyVersion generateEncryptedKey(String encryptionKeyName)
+ throws IOException, GeneralSecurityException {
+ doAccessCheck(encryptionKeyName, KeyOpType.GENERATE_EEK);
+ return provider.generateEncryptedKey(encryptionKeyName);
+ }
+
+ @Override
+ public KeyVersion decryptEncryptedKey(EncryptedKeyVersion encryptedKeyVersion)
+ throws IOException, GeneralSecurityException {
+ doAccessCheck(
+ encryptedKeyVersion.getEncryptionKeyName(), KeyOpType.DECRYPT_EEK);
+ return provider.decryptEncryptedKey(encryptedKeyVersion);
+ }
+
+ @Override
+ public KeyVersion getKeyVersion(String versionName) throws IOException {
+ KeyVersion keyVersion = provider.getKeyVersion(versionName);
+ if (keyVersion != null) {
+ doAccessCheck(keyVersion.getName(), KeyOpType.READ);
+ }
+ return keyVersion;
+ }
+
+ @Override
+ public List<String> getKeys() throws IOException {
+ return provider.getKeys();
+ }
+
+ @Override
+ public List<KeyVersion> getKeyVersions(String name) throws IOException {
+ doAccessCheck(name, KeyOpType.READ);
+ return provider.getKeyVersions(name);
+ }
+
+ @Override
+ public Metadata getMetadata(String name) throws IOException {
+ doAccessCheck(name, KeyOpType.READ);
+ return provider.getMetadata(name);
+ }
+
+ @Override
+ public Metadata[] getKeysMetadata(String... names) throws IOException {
+ for (String name : names) {
+ doAccessCheck(name, KeyOpType.READ);
+ }
+ return provider.getKeysMetadata(names);
+ }
+
+ @Override
+ public KeyVersion getCurrentKey(String name) throws IOException {
+ doAccessCheck(name, KeyOpType.READ);
+ return provider.getCurrentKey(name);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ provider.flush();
+ }
+
+ @Override
+ public boolean isTransient() {
+ return provider.isTransient();
+ }
+
+ private void doAccessCheck(String keyName, KeyOpType opType) throws
+ IOException {
+ Metadata metadata = provider.getMetadata(keyName);
+ if (metadata != null) {
+ String aclName = metadata.getAttributes().get(KEY_ACL_NAME);
+ checkAccess((aclName == null) ? keyName : aclName, getUser(), opType);
+ }
+ }
+
+ private UserGroupInformation getUser() throws IOException {
+ return UserGroupInformation.getCurrentUser();
+ }
+
+ @Override
+ protected KeyProvider getKeyProvider() {
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return provider.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b02a4b40/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index e947c9b..c76ca3b 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -443,6 +443,112 @@ $ keytool -genkey -alias tomcat -keyalg RSA
+---+
+*** Key Access Control
+
+ KMS supports access control for all non-read operations at the Key level.
+ All Key Access operations are classified as :
+
+ * MANAGEMENT - createKey, deleteKey, rolloverNewVersion
+
+ * GENERATE_EEK - generateEncryptedKey, warmUpEncryptedKeys
+
+ * DECRYPT_EEK - decryptEncryptedKey;
+
+ * READ - getKeyVersion, getKeyVersions, getMetadata, getKeysMetadata,
+ getCurrentKey;
+
+ * ALL - all of the above;
+
+ These can be defined in the KMS <<<etc/hadoop/kms-acls.xml>>> as follows
+
+ For all keys for which a key access has not been explicitly configured, It
+ is possible to configure a default key access control for a subset of the
+ operation types.
+
+ If no ACL is configured for a specific key AND no default ACL is configured
+ for the requested operation, then access will be DENIED.
+
+ <<NOTE:>> The default ACL does not support <<<ALL>>> operation qualifier.
+
++---+
+ <property>
+ <name>key.acl.testKey1.MANAGEMENT</name>
+ <value>*</value>
+ <description>
+ ACL for create-key, deleteKey and rolloverNewVersion operations.
+ </description>
+ </property>
+
+ <property>
+ <name>key.acl.testKey2.GENERATE_EEK</name>
+ <value>*</value>
+ <description>
+ ACL for generateEncryptedKey operations.
+ </description>
+ </property>
+
+ <property>
+ <name>key.acl.testKey3.DECRYPT_EEK</name>
+ <value>*</value>
+ <description>
+ ACL for decryptEncryptedKey operations.
+ </description>
+ </property>
+
+ <property>
+ <name>key.acl.testKey4.READ</name>
+ <value>*</value>
+ <description>
+ ACL for getKeyVersion, getKeyVersions, getMetadata, getKeysMetadata,
+ getCurrentKey operations
+ </description>
+ </property>
+
+ <property>
+ <name>key.acl.testKey5.ALL</name>
+ <value>*</value>
+ <description>
+ ACL for ALL operations.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.MANAGEMENT</name>
+ <value>user1,user2</value>
+ <description>
+ default ACL for MANAGEMENT operations for all keys that are not
+ explicitly defined.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.GENERATE_EEK</name>
+ <value>user1,user2</value>
+ <description>
+ default ACL for GENERATE_EEK operations for all keys that are not
+ explicitly defined.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.DECRYPT_EEK</name>
+ <value>user1,user2</value>
+ <description>
+ default ACL for DECRYPT_EEK operations for all keys that are not
+ explicitly defined.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.READ</name>
+ <value>user1,user2</value>
+ <description>
+ default ACL for READ operations for all keys that are not
+ explicitly defined.
+ </description>
+ </property>
++---+
+
** KMS Delegation Token Configuration
KMS delegation token secret manager can be configured with the following
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b02a4b40/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 74eab5c..1ca0c0d 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.crypto.key.kms.server;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
+import org.apache.hadoop.crypto.key.KeyProvider.Options;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
@@ -338,6 +340,13 @@ public class TestKMS {
UserGroupInformation.setConfiguration(conf);
File confDir = getTestDir();
conf = createBaseKMSConf(confDir);
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k1.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k2.MANAGEMENT", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k2.READ", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k3.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k4.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k5.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k6.ALL", "*");
writeConf(confDir, conf);
runServer(null, null, confDir, new KMSCallable() {
@@ -492,10 +501,20 @@ public class TestKMS {
options = new KeyProvider.Options(conf);
options.setCipher("AES/CTR/NoPadding");
options.setBitLength(128);
- kp.createKey("k2", options);
+ KeyVersion kVer2 = kp.createKey("k2", options);
KeyProvider.Metadata meta = kp.getMetadata("k2");
Assert.assertNull(meta.getDescription());
- Assert.assertTrue(meta.getAttributes().isEmpty());
+ Assert.assertEquals("k2", meta.getAttributes().get("key.acl.name"));
+
+ // test key ACL.. k2 is granted only MANAGEMENT Op access
+ try {
+ kpExt =
+ KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
+ kpExt.generateEncryptedKey(kVer2.getName());
+ Assert.fail("User should not be allowed to encrypt !!");
+ } catch (Exception ex) {
+ //
+ }
// createKey() description, no tags
options = new KeyProvider.Options(conf);
@@ -505,7 +524,7 @@ public class TestKMS {
kp.createKey("k3", options);
meta = kp.getMetadata("k3");
Assert.assertEquals("d", meta.getDescription());
- Assert.assertTrue(meta.getAttributes().isEmpty());
+ Assert.assertEquals("k3", meta.getAttributes().get("key.acl.name"));
Map<String, String> attributes = new HashMap<String, String>();
attributes.put("a", "A");
@@ -514,6 +533,7 @@ public class TestKMS {
options = new KeyProvider.Options(conf);
options.setCipher("AES/CTR/NoPadding");
options.setBitLength(128);
+ attributes.put("key.acl.name", "k4");
options.setAttributes(attributes);
kp.createKey("k4", options);
meta = kp.getMetadata("k4");
@@ -525,6 +545,7 @@ public class TestKMS {
options.setCipher("AES/CTR/NoPadding");
options.setBitLength(128);
options.setDescription("d");
+ attributes.put("key.acl.name", "k5");
options.setAttributes(attributes);
kp.createKey("k5", options);
meta = kp.getMetadata("k5");
@@ -565,6 +586,201 @@ public class TestKMS {
}
@Test
+ public void testKeyACLs() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set("hadoop.security.authentication", "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ final File testDir = getTestDir();
+ conf = createBaseKMSConf(testDir);
+ conf.set("hadoop.kms.authentication.type", "kerberos");
+ conf.set("hadoop.kms.authentication.kerberos.keytab",
+ keytab.getAbsolutePath());
+ conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
+ conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
+
+ for (KMSACLs.Type type : KMSACLs.Type.values()) {
+ conf.set(type.getAclConfigKey(), type.toString());
+ }
+ conf.set(KMSACLs.Type.CREATE.getAclConfigKey(),"CREATE,ROLLOVER,GET,SET_KEY_MATERIAL,GENERATE_EEK");
+ conf.set(KMSACLs.Type.ROLLOVER.getAclConfigKey(),"CREATE,ROLLOVER,GET,SET_KEY_MATERIAL,GENERATE_EEK");
+ conf.set(KMSACLs.Type.GENERATE_EEK.getAclConfigKey(),"CREATE,ROLLOVER,GET,SET_KEY_MATERIAL,GENERATE_EEK");
+ conf.set(KMSACLs.Type.DECRYPT_EEK.getAclConfigKey(),"CREATE,ROLLOVER,GET,SET_KEY_MATERIAL,GENERATE_EEK");
+
+
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "test_key.MANAGEMENT", "CREATE");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "all_access.ALL", "GENERATE_EEK");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "all_access.DECRYPT_EEK", "ROLLOVER");
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT", "ROLLOVER");
+
+ writeConf(testDir, conf);
+
+ runServer(null, null, testDir, new KMSCallable() {
+
+ @Override
+ public Void call() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
+ final URI uri = createKMSUri(getKMSUrl());
+
+ doAs("CREATE", new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ KeyProvider kp = new KMSClientProvider(uri, conf);
+ try {
+ Options options = new KeyProvider.Options(conf);
+ Map<String, String> attributes = options.getAttributes();
+ HashMap<String,String> newAttribs = new HashMap<String, String>(attributes);
+ newAttribs.put("key.acl.name", "test_key");
+ options.setAttributes(newAttribs);
+ KeyProvider.KeyVersion kv = kp.createKey("k0", options);
+ Assert.assertNull(kv.getMaterial());
+ KeyVersion rollVersion = kp.rollNewVersion("k0");
+ Assert.assertNull(rollVersion.getMaterial());
+ KeyProviderCryptoExtension kpce =
+ KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
+ try {
+ kpce.generateEncryptedKey("k0");
+ Assert.fail("User [CREATE] should not be allowed to generate_eek on k0");
+ } catch (Exception e) {
+ // Ignore
+ }
+ newAttribs = new HashMap<String, String>(attributes);
+ newAttribs.put("key.acl.name", "all_access");
+ options.setAttributes(newAttribs);
+ try {
+ kp.createKey("kx", options);
+ Assert.fail("User [CREATE] should not be allowed to create kx");
+ } catch (Exception e) {
+ // Ignore
+ }
+ } catch (Exception ex) {
+ Assert.fail(ex.getMessage());
+ }
+ return null;
+ }
+ });
+
+ doAs("ROLLOVER", new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ KeyProvider kp = new KMSClientProvider(uri, conf);
+ try {
+ Options options = new KeyProvider.Options(conf);
+ Map<String, String> attributes = options.getAttributes();
+ HashMap<String,String> newAttribs = new HashMap<String, String>(attributes);
+ newAttribs.put("key.acl.name", "test_key2");
+ options.setAttributes(newAttribs);
+ KeyProvider.KeyVersion kv = kp.createKey("k1", options);
+ Assert.assertNull(kv.getMaterial());
+ KeyVersion rollVersion = kp.rollNewVersion("k1");
+ Assert.assertNull(rollVersion.getMaterial());
+ try {
+ kp.rollNewVersion("k0");
+ Assert.fail("User [ROLLOVER] should not be allowed to rollover k0");
+ } catch (Exception e) {
+ // Ignore
+ }
+ KeyProviderCryptoExtension kpce =
+ KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
+ try {
+ kpce.generateEncryptedKey("k1");
+ Assert.fail("User [ROLLOVER] should not be allowed to generate_eek on k1");
+ } catch (Exception e) {
+ // Ignore
+ }
+ newAttribs = new HashMap<String, String>(attributes);
+ newAttribs.put("key.acl.name", "all_access");
+ options.setAttributes(newAttribs);
+ try {
+ kp.createKey("kx", options);
+ Assert.fail("User [ROLLOVER] should not be allowed to create kx");
+ } catch (Exception e) {
+ // Ignore
+ }
+ } catch (Exception ex) {
+ Assert.fail(ex.getMessage());
+ }
+ return null;
+ }
+ });
+
+ doAs("GET", new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ KeyProvider kp = new KMSClientProvider(uri, conf);
+ try {
+ Options options = new KeyProvider.Options(conf);
+ Map<String, String> attributes = options.getAttributes();
+ HashMap<String,String> newAttribs = new HashMap<String, String>(attributes);
+ newAttribs.put("key.acl.name", "test_key");
+ options.setAttributes(newAttribs);
+ try {
+ kp.createKey("k2", options);
+ Assert.fail("User [GET] should not be allowed to create key..");
+ } catch (Exception e) {
+ // Ignore
+ }
+ newAttribs = new HashMap<String, String>(attributes);
+ newAttribs.put("key.acl.name", "all_access");
+ options.setAttributes(newAttribs);
+ try {
+ kp.createKey("kx", options);
+ Assert.fail("User [GET] should not be allowed to create kx");
+ } catch (Exception e) {
+ // Ignore
+ }
+ } catch (Exception ex) {
+ Assert.fail(ex.getMessage());
+ }
+ return null;
+ }
+ });
+
+ final EncryptedKeyVersion ekv = doAs("GENERATE_EEK", new PrivilegedExceptionAction<EncryptedKeyVersion>() {
+ @Override
+ public EncryptedKeyVersion run() throws Exception {
+ KeyProvider kp = new KMSClientProvider(uri, conf);
+ try {
+ Options options = new KeyProvider.Options(conf);
+ Map<String, String> attributes = options.getAttributes();
+ HashMap<String,String> newAttribs = new HashMap<String, String>(attributes);
+ newAttribs.put("key.acl.name", "all_access");
+ options.setAttributes(newAttribs);
+ kp.createKey("kx", options);
+ KeyProviderCryptoExtension kpce =
+ KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
+ try {
+ return kpce.generateEncryptedKey("kx");
+ } catch (Exception e) {
+ Assert.fail("User [GENERATE_EEK] should be allowed to generate_eek on kx");
+ }
+ } catch (Exception ex) {
+ Assert.fail(ex.getMessage());
+ }
+ return null;
+ }
+ });
+
+ doAs("ROLLOVER", new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ KeyProvider kp = new KMSClientProvider(uri, conf);
+ try {
+ KeyProviderCryptoExtension kpce =
+ KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
+ kpce.decryptEncryptedKey(ekv);
+ } catch (Exception ex) {
+ Assert.fail(ex.getMessage());
+ }
+ return null;
+ }
+ });
+ return null;
+ }
+ });
+ }
+
+ @Test
public void testACLs() throws Exception {
Configuration conf = new Configuration();
conf.set("hadoop.security.authentication", "kerberos");
@@ -586,6 +802,9 @@ public class TestKMS {
conf.set(KMSACLs.Type.ROLLOVER.getAclConfigKey(),
KMSACLs.Type.ROLLOVER.toString() + ",SET_KEY_MATERIAL");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k0.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k1.ALL", "*");
+
writeConf(testDir, conf);
runServer(null, null, testDir, new KMSCallable() {
@@ -891,6 +1110,9 @@ public class TestKMS {
conf.set(KMSACLs.Type.DECRYPT_EEK.getAclConfigKey(), "client,hdfs,otheradmin");
conf.set(KMSACLs.Type.DECRYPT_EEK.getBlacklistConfigKey(), "hdfs,otheradmin");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "ck0.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "ck1.ALL", "*");
+
writeConf(testDir, conf);
runServer(null, null, testDir, new KMSCallable() {
@@ -973,6 +1195,7 @@ public class TestKMS {
conf.set(type.getAclConfigKey(), " ");
}
conf.set(KMSACLs.Type.CREATE.getAclConfigKey(), "client");
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT", "client,client/host");
writeConf(testDir, conf);
@@ -1096,6 +1319,9 @@ public class TestKMS {
conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kA.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kD.ALL", "*");
+
writeConf(testDir, conf);
runServer(null, null, testDir, new KMSCallable() {
@@ -1164,6 +1390,10 @@ public class TestKMS {
conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
conf.set("hadoop.kms.proxyuser.client.users", "foo");
conf.set("hadoop.kms.proxyuser.client.hosts", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kAA.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kBB.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kCC.ALL", "*");
+
writeConf(testDir, conf);
runServer(null, null, testDir, new KMSCallable() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b02a4b40/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKeyAuthorizationKeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKeyAuthorizationKeyProvider.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKeyAuthorizationKeyProvider.java
new file mode 100644
index 0000000..a79926a
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKeyAuthorizationKeyProvider.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.URI;
+import java.security.PrivilegedExceptionAction;
+import java.security.SecureRandom;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
+import org.apache.hadoop.crypto.key.KeyProvider.Options;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
+import org.apache.hadoop.crypto.key.UserProvider;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyACLs;
+import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyOpType;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestKeyAuthorizationKeyProvider {
+
+ private static final String CIPHER = "AES";
+
+ @Test
+ public void testCreateKey() throws Exception {
+ final Configuration conf = new Configuration();
+ KeyProvider kp =
+ new UserProvider.Factory().createProvider(new URI("user:///"), conf);
+ KeyACLs mock = mock(KeyACLs.class);
+ when(mock.isACLPresent("foo", KeyOpType.MANAGEMENT)).thenReturn(true);
+ UserGroupInformation u1 = UserGroupInformation.createRemoteUser("u1");
+ when(mock.hasAccessToKey("foo", u1, KeyOpType.MANAGEMENT)).thenReturn(true);
+ final KeyProviderCryptoExtension kpExt =
+ new KeyAuthorizationKeyProvider(
+ KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp),
+ mock);
+
+ u1.doAs(
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ kpExt.createKey("foo", SecureRandom.getSeed(16),
+ newOptions(conf));
+ } catch (IOException ioe) {
+ Assert.fail("User should be Authorized !!");
+ }
+
+ // "bar" key not configured
+ try {
+ kpExt.createKey("bar", SecureRandom.getSeed(16),
+ newOptions(conf));
+ Assert.fail("User should NOT be Authorized !!");
+ } catch (IOException ioe) {
+ // Ignore
+ }
+ return null;
+ }
+ }
+ );
+
+ // Unauthorized User
+ UserGroupInformation.createRemoteUser("badGuy").doAs(
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ kpExt.createKey("foo", SecureRandom.getSeed(16),
+ newOptions(conf));
+ Assert.fail("User should NOT be Authorized !!");
+ } catch (IOException ioe) {
+ // Ignore
+ }
+ return null;
+ }
+ }
+ );
+ }
+
+ @Test
+ public void testOpsWhenACLAttributeExists() throws Exception {
+ final Configuration conf = new Configuration();
+ KeyProvider kp =
+ new UserProvider.Factory().createProvider(new URI("user:///"), conf);
+ KeyACLs mock = mock(KeyACLs.class);
+ when(mock.isACLPresent("testKey", KeyOpType.MANAGEMENT)).thenReturn(true);
+ when(mock.isACLPresent("testKey", KeyOpType.GENERATE_EEK)).thenReturn(true);
+ when(mock.isACLPresent("testKey", KeyOpType.DECRYPT_EEK)).thenReturn(true);
+ when(mock.isACLPresent("testKey", KeyOpType.ALL)).thenReturn(true);
+ UserGroupInformation u1 = UserGroupInformation.createRemoteUser("u1");
+ UserGroupInformation u2 = UserGroupInformation.createRemoteUser("u2");
+ UserGroupInformation u3 = UserGroupInformation.createRemoteUser("u3");
+ UserGroupInformation sudo = UserGroupInformation.createRemoteUser("sudo");
+ when(mock.hasAccessToKey("testKey", u1, KeyOpType.MANAGEMENT)).thenReturn(true);
+ when(mock.hasAccessToKey("testKey", u2, KeyOpType.GENERATE_EEK)).thenReturn(true);
+ when(mock.hasAccessToKey("testKey", u3, KeyOpType.DECRYPT_EEK)).thenReturn(true);
+ when(mock.hasAccessToKey("testKey", sudo, KeyOpType.ALL)).thenReturn(true);
+ final KeyProviderCryptoExtension kpExt =
+ new KeyAuthorizationKeyProvider(
+ KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp),
+ mock);
+
+ final KeyVersion barKv = u1.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ Options opt = newOptions(conf);
+ Map<String, String> m = new HashMap<String, String>();
+ m.put("key.acl.name", "testKey");
+ opt.setAttributes(m);
+ try {
+ KeyVersion kv =
+ kpExt.createKey("foo", SecureRandom.getSeed(16), opt);
+ kpExt.rollNewVersion(kv.getName());
+ kpExt.rollNewVersion(kv.getName(), SecureRandom.getSeed(16));
+ kpExt.deleteKey(kv.getName());
+ } catch (IOException ioe) {
+ Assert.fail("User should be Authorized !!");
+ }
+
+ KeyVersion retkv = null;
+ try {
+ retkv = kpExt.createKey("bar", SecureRandom.getSeed(16), opt);
+ kpExt.generateEncryptedKey(retkv.getName());
+ Assert.fail("User should NOT be Authorized to generate EEK !!");
+ } catch (IOException ioe) {
+ }
+ Assert.assertNotNull(retkv);
+ return retkv;
+ }
+ }
+ );
+
+ final EncryptedKeyVersion barEKv =
+ u2.doAs(
+ new PrivilegedExceptionAction<EncryptedKeyVersion>() {
+ @Override
+ public EncryptedKeyVersion run() throws Exception {
+ try {
+ kpExt.deleteKey(barKv.getName());
+ Assert.fail("User should NOT be Authorized to "
+ + "perform any other operation !!");
+ } catch (IOException ioe) {
+ }
+ return kpExt.generateEncryptedKey(barKv.getName());
+ }
+ });
+
+ u3.doAs(
+ new PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ try {
+ kpExt.deleteKey(barKv.getName());
+ Assert.fail("User should NOT be Authorized to "
+ + "perform any other operation !!");
+ } catch (IOException ioe) {
+ }
+ return kpExt.decryptEncryptedKey(barEKv);
+ }
+ });
+
+ sudo.doAs(
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ Options opt = newOptions(conf);
+ Map<String, String> m = new HashMap<String, String>();
+ m.put("key.acl.name", "testKey");
+ opt.setAttributes(m);
+ try {
+ KeyVersion kv =
+ kpExt.createKey("foo", SecureRandom.getSeed(16), opt);
+ kpExt.rollNewVersion(kv.getName());
+ kpExt.rollNewVersion(kv.getName(), SecureRandom.getSeed(16));
+ EncryptedKeyVersion ekv = kpExt.generateEncryptedKey(kv.getName());
+ kpExt.decryptEncryptedKey(ekv);
+ kpExt.deleteKey(kv.getName());
+ } catch (IOException ioe) {
+ Assert.fail("User should be Allowed to do everything !!");
+ }
+ return null;
+ }
+ }
+ );
+ }
+
+ private static KeyProvider.Options newOptions(Configuration conf) {
+ KeyProvider.Options options = new KeyProvider.Options(conf);
+ options.setCipher(CIPHER);
+ options.setBitLength(128);
+ return options;
+ }
+
+}
[12/20] HADOOP-11074. Move s3-related FS connector code to
hadoop-aws. (David S. Wang via Colin Patrick McCabe)
Posted by sz...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
new file mode 100644
index 0000000..acc5500
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
@@ -0,0 +1,796 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.BufferedOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BufferedFSInputStream;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.s3.S3Exception;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryProxy;
+import org.apache.hadoop.util.Progressable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * A {@link FileSystem} for reading and writing files stored on
+ * <a href="http://aws.amazon.com/s3">Amazon S3</a>.
+ * Unlike {@link org.apache.hadoop.fs.s3.S3FileSystem} this implementation
+ * stores files on S3 in their
+ * native form so they can be read by other S3 tools.
+ *
+ * A note about directories. S3 of course has no "native" support for them.
+ * The idiom we choose then is: for any directory created by this class,
+ * we use an empty object "#{dirpath}_$folder$" as a marker.
+ * Further, to interoperate with other S3 tools, we also accept the following:
+ * - an object "#{dirpath}/' denoting a directory marker
+ * - if there exists any objects with the prefix "#{dirpath}/", then the
+ * directory is said to exist
+ * - if both a file with the name of a directory and a marker for that
+ * directory exists, then the *file masks the directory*, and the directory
+ * is never returned.
+ * </p>
+ * @see org.apache.hadoop.fs.s3.S3FileSystem
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class NativeS3FileSystem extends FileSystem {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(NativeS3FileSystem.class);
+
+ private static final String FOLDER_SUFFIX = "_$folder$";
+ static final String PATH_DELIMITER = Path.SEPARATOR;
+ private static final int S3_MAX_LISTING_LENGTH = 1000;
+
+ static class NativeS3FsInputStream extends FSInputStream {
+
+ private NativeFileSystemStore store;
+ private Statistics statistics;
+ private InputStream in;
+ private final String key;
+ private long pos = 0;
+
+ public NativeS3FsInputStream(NativeFileSystemStore store, Statistics statistics, InputStream in, String key) {
+ Preconditions.checkNotNull(in, "Null input stream");
+ this.store = store;
+ this.statistics = statistics;
+ this.in = in;
+ this.key = key;
+ }
+
+ @Override
+ public synchronized int read() throws IOException {
+ int result;
+ try {
+ result = in.read();
+ } catch (IOException e) {
+ LOG.info("Received IOException while reading '{}', attempting to reopen",
+ key);
+ LOG.debug("{}", e, e);
+ try {
+ seek(pos);
+ result = in.read();
+ } catch (EOFException eof) {
+ LOG.debug("EOF on input stream read: {}", eof, eof);
+ result = -1;
+ }
+ }
+ if (result != -1) {
+ pos++;
+ }
+ if (statistics != null && result != -1) {
+ statistics.incrementBytesRead(1);
+ }
+ return result;
+ }
+ @Override
+ public synchronized int read(byte[] b, int off, int len)
+ throws IOException {
+ if (in == null) {
+ throw new EOFException("Cannot read closed stream");
+ }
+ int result = -1;
+ try {
+ result = in.read(b, off, len);
+ } catch (EOFException eof) {
+ throw eof;
+ } catch (IOException e) {
+ LOG.info( "Received IOException while reading '{}'," +
+ " attempting to reopen.", key);
+ seek(pos);
+ result = in.read(b, off, len);
+ }
+ if (result > 0) {
+ pos += result;
+ }
+ if (statistics != null && result > 0) {
+ statistics.incrementBytesRead(result);
+ }
+ return result;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ closeInnerStream();
+ }
+
+ /**
+ * Close the inner stream if not null. Even if an exception
+ * is raised during the close, the field is set to null
+ * @throws IOException if raised by the close() operation.
+ */
+ private void closeInnerStream() throws IOException {
+ if (in != null) {
+ try {
+ in.close();
+ } finally {
+ in = null;
+ }
+ }
+ }
+
+ /**
+ * Update inner stream with a new stream and position
+ * @param newStream new stream -must not be null
+ * @param newpos new position
+ * @throws IOException IO exception on a failure to close the existing
+ * stream.
+ */
+ private synchronized void updateInnerStream(InputStream newStream, long newpos) throws IOException {
+ Preconditions.checkNotNull(newStream, "Null newstream argument");
+ closeInnerStream();
+ in = newStream;
+ this.pos = newpos;
+ }
+
+ @Override
+ public synchronized void seek(long newpos) throws IOException {
+ if (newpos < 0) {
+ throw new EOFException(
+ FSExceptionMessages.NEGATIVE_SEEK);
+ }
+ if (pos != newpos) {
+ // the seek is attempting to move the current position
+ LOG.debug("Opening key '{}' for reading at position '{}", key, newpos);
+ InputStream newStream = store.retrieve(key, newpos);
+ updateInnerStream(newStream, newpos);
+ }
+ }
+
+ @Override
+ public synchronized long getPos() throws IOException {
+ return pos;
+ }
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ return false;
+ }
+ }
+
+ private class NativeS3FsOutputStream extends OutputStream {
+
+ private Configuration conf;
+ private String key;
+ private File backupFile;
+ private OutputStream backupStream;
+ private MessageDigest digest;
+ private boolean closed;
+ private LocalDirAllocator lDirAlloc;
+
+ public NativeS3FsOutputStream(Configuration conf,
+ NativeFileSystemStore store, String key, Progressable progress,
+ int bufferSize) throws IOException {
+ this.conf = conf;
+ this.key = key;
+ this.backupFile = newBackupFile();
+ LOG.info("OutputStream for key '" + key + "' writing to tempfile '" + this.backupFile + "'");
+ try {
+ this.digest = MessageDigest.getInstance("MD5");
+ this.backupStream = new BufferedOutputStream(new DigestOutputStream(
+ new FileOutputStream(backupFile), this.digest));
+ } catch (NoSuchAlgorithmException e) {
+ LOG.warn("Cannot load MD5 digest algorithm," +
+ "skipping message integrity check.", e);
+ this.backupStream = new BufferedOutputStream(
+ new FileOutputStream(backupFile));
+ }
+ }
+
+ private File newBackupFile() throws IOException {
+ if (lDirAlloc == null) {
+ lDirAlloc = new LocalDirAllocator("fs.s3.buffer.dir");
+ }
+ File result = lDirAlloc.createTmpFileForWrite("output-", LocalDirAllocator.SIZE_UNKNOWN, conf);
+ result.deleteOnExit();
+ return result;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ backupStream.flush();
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (closed) {
+ return;
+ }
+
+ backupStream.close();
+ LOG.info("OutputStream for key '{}' closed. Now beginning upload", key);
+
+ try {
+ byte[] md5Hash = digest == null ? null : digest.digest();
+ store.storeFile(key, backupFile, md5Hash);
+ } finally {
+ if (!backupFile.delete()) {
+ LOG.warn("Could not delete temporary s3n file: " + backupFile);
+ }
+ super.close();
+ closed = true;
+ }
+ LOG.info("OutputStream for key '{}' upload complete", key);
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ backupStream.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ backupStream.write(b, off, len);
+ }
+ }
+
+ private URI uri;
+ private NativeFileSystemStore store;
+ private Path workingDir;
+
+ public NativeS3FileSystem() {
+ // set store in initialize()
+ }
+
+ public NativeS3FileSystem(NativeFileSystemStore store) {
+ this.store = store;
+ }
+
+ /**
+ * Return the protocol scheme for the FileSystem.
+ * <p/>
+ *
+ * @return <code>s3n</code>
+ */
+ @Override
+ public String getScheme() {
+ return "s3n";
+ }
+
+ @Override
+ public void initialize(URI uri, Configuration conf) throws IOException {
+ super.initialize(uri, conf);
+ if (store == null) {
+ store = createDefaultStore(conf);
+ }
+ store.initialize(uri, conf);
+ setConf(conf);
+ this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
+ this.workingDir =
+ new Path("/user", System.getProperty("user.name")).makeQualified(this.uri, this.getWorkingDirectory());
+ }
+
+ private static NativeFileSystemStore createDefaultStore(Configuration conf) {
+ NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
+
+ RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
+ conf.getInt("fs.s3.maxRetries", 4),
+ conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
+ Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
+ new HashMap<Class<? extends Exception>, RetryPolicy>();
+ exceptionToPolicyMap.put(IOException.class, basePolicy);
+ exceptionToPolicyMap.put(S3Exception.class, basePolicy);
+
+ RetryPolicy methodPolicy = RetryPolicies.retryByException(
+ RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
+ Map<String, RetryPolicy> methodNameToPolicyMap =
+ new HashMap<String, RetryPolicy>();
+ methodNameToPolicyMap.put("storeFile", methodPolicy);
+ methodNameToPolicyMap.put("rename", methodPolicy);
+
+ return (NativeFileSystemStore)
+ RetryProxy.create(NativeFileSystemStore.class, store,
+ methodNameToPolicyMap);
+ }
+
+ private static String pathToKey(Path path) {
+ if (path.toUri().getScheme() != null && path.toUri().getPath().isEmpty()) {
+ // allow uris without trailing slash after bucket to refer to root,
+ // like s3n://mybucket
+ return "";
+ }
+ if (!path.isAbsolute()) {
+ throw new IllegalArgumentException("Path must be absolute: " + path);
+ }
+ String ret = path.toUri().getPath().substring(1); // remove initial slash
+ if (ret.endsWith("/") && (ret.indexOf("/") != ret.length() - 1)) {
+ ret = ret.substring(0, ret.length() -1);
+ }
+ return ret;
+ }
+
+ private static Path keyToPath(String key) {
+ return new Path("/" + key);
+ }
+
+ private Path makeAbsolute(Path path) {
+ if (path.isAbsolute()) {
+ return path;
+ }
+ return new Path(workingDir, path);
+ }
+
+ /** This optional operation is not yet supported. */
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new IOException("Not supported");
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize, short replication, long blockSize,
+ Progressable progress) throws IOException {
+
+ if (exists(f) && !overwrite) {
+ throw new FileAlreadyExistsException("File already exists: " + f);
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Creating new file '" + f + "' in S3");
+ }
+ Path absolutePath = makeAbsolute(f);
+ String key = pathToKey(absolutePath);
+ return new FSDataOutputStream(new NativeS3FsOutputStream(getConf(), store,
+ key, progress, bufferSize), statistics);
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recurse) throws IOException {
+ FileStatus status;
+ try {
+ status = getFileStatus(f);
+ } catch (FileNotFoundException e) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Delete called for '" + f +
+ "' but file does not exist, so returning false");
+ }
+ return false;
+ }
+ Path absolutePath = makeAbsolute(f);
+ String key = pathToKey(absolutePath);
+ if (status.isDirectory()) {
+ if (!recurse && listStatus(f).length > 0) {
+ throw new IOException("Can not delete " + f + " as is a not empty directory and recurse option is false");
+ }
+
+ createParent(f);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Deleting directory '" + f + "'");
+ }
+ String priorLastKey = null;
+ do {
+ PartialListing listing = store.list(key, S3_MAX_LISTING_LENGTH, priorLastKey, true);
+ for (FileMetadata file : listing.getFiles()) {
+ store.delete(file.getKey());
+ }
+ priorLastKey = listing.getPriorLastKey();
+ } while (priorLastKey != null);
+
+ try {
+ store.delete(key + FOLDER_SUFFIX);
+ } catch (FileNotFoundException e) {
+ //this is fine, we don't require a marker
+ }
+ } else {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Deleting file '" + f + "'");
+ }
+ createParent(f);
+ store.delete(key);
+ }
+ return true;
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ Path absolutePath = makeAbsolute(f);
+ String key = pathToKey(absolutePath);
+
+ if (key.length() == 0) { // root always exists
+ return newDirectory(absolutePath);
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("getFileStatus retrieving metadata for key '" + key + "'");
+ }
+ FileMetadata meta = store.retrieveMetadata(key);
+ if (meta != null) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("getFileStatus returning 'file' for key '" + key + "'");
+ }
+ return newFile(meta, absolutePath);
+ }
+ if (store.retrieveMetadata(key + FOLDER_SUFFIX) != null) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("getFileStatus returning 'directory' for key '" + key +
+ "' as '" + key + FOLDER_SUFFIX + "' exists");
+ }
+ return newDirectory(absolutePath);
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("getFileStatus listing key '" + key + "'");
+ }
+ PartialListing listing = store.list(key, 1);
+ if (listing.getFiles().length > 0 ||
+ listing.getCommonPrefixes().length > 0) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("getFileStatus returning 'directory' for key '" + key +
+ "' as it has contents");
+ }
+ return newDirectory(absolutePath);
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("getFileStatus could not find key '" + key + "'");
+ }
+ throw new FileNotFoundException("No such file or directory '" + absolutePath + "'");
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ /**
+ * <p>
+ * If <code>f</code> is a file, this method will make a single call to S3.
+ * If <code>f</code> is a directory, this method will make a maximum of
+ * (<i>n</i> / 1000) + 2 calls to S3, where <i>n</i> is the total number of
+ * files and directories contained directly in <code>f</code>.
+ * </p>
+ */
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+
+ Path absolutePath = makeAbsolute(f);
+ String key = pathToKey(absolutePath);
+
+ if (key.length() > 0) {
+ FileMetadata meta = store.retrieveMetadata(key);
+ if (meta != null) {
+ return new FileStatus[] { newFile(meta, absolutePath) };
+ }
+ }
+
+ URI pathUri = absolutePath.toUri();
+ Set<FileStatus> status = new TreeSet<FileStatus>();
+ String priorLastKey = null;
+ do {
+ PartialListing listing = store.list(key, S3_MAX_LISTING_LENGTH, priorLastKey, false);
+ for (FileMetadata fileMetadata : listing.getFiles()) {
+ Path subpath = keyToPath(fileMetadata.getKey());
+ String relativePath = pathUri.relativize(subpath.toUri()).getPath();
+
+ if (fileMetadata.getKey().equals(key + "/")) {
+ // this is just the directory we have been asked to list
+ }
+ else if (relativePath.endsWith(FOLDER_SUFFIX)) {
+ status.add(newDirectory(new Path(
+ absolutePath,
+ relativePath.substring(0, relativePath.indexOf(FOLDER_SUFFIX)))));
+ }
+ else {
+ status.add(newFile(fileMetadata, subpath));
+ }
+ }
+ for (String commonPrefix : listing.getCommonPrefixes()) {
+ Path subpath = keyToPath(commonPrefix);
+ String relativePath = pathUri.relativize(subpath.toUri()).getPath();
+ status.add(newDirectory(new Path(absolutePath, relativePath)));
+ }
+ priorLastKey = listing.getPriorLastKey();
+ } while (priorLastKey != null);
+
+ if (status.isEmpty() &&
+ key.length() > 0 &&
+ store.retrieveMetadata(key + FOLDER_SUFFIX) == null) {
+ throw new FileNotFoundException("File " + f + " does not exist.");
+ }
+
+ return status.toArray(new FileStatus[status.size()]);
+ }
+
+ private FileStatus newFile(FileMetadata meta, Path path) {
+ return new FileStatus(meta.getLength(), false, 1, getDefaultBlockSize(),
+ meta.getLastModified(), path.makeQualified(this.getUri(), this.getWorkingDirectory()));
+ }
+
+ private FileStatus newDirectory(Path path) {
+ return new FileStatus(0, true, 1, 0, 0, path.makeQualified(this.getUri(), this.getWorkingDirectory()));
+ }
+
+ @Override
+ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ Path absolutePath = makeAbsolute(f);
+ List<Path> paths = new ArrayList<Path>();
+ do {
+ paths.add(0, absolutePath);
+ absolutePath = absolutePath.getParent();
+ } while (absolutePath != null);
+
+ boolean result = true;
+ for (Path path : paths) {
+ result &= mkdir(path);
+ }
+ return result;
+ }
+
+ private boolean mkdir(Path f) throws IOException {
+ try {
+ FileStatus fileStatus = getFileStatus(f);
+ if (fileStatus.isFile()) {
+ throw new FileAlreadyExistsException(String.format(
+ "Can't make directory for path '%s' since it is a file.", f));
+
+ }
+ } catch (FileNotFoundException e) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Making dir '" + f + "' in S3");
+ }
+ String key = pathToKey(f) + FOLDER_SUFFIX;
+ store.storeEmptyFile(key);
+ }
+ return true;
+ }
+
+ @Override
+ public FSDataInputStream open(Path f, int bufferSize) throws IOException {
+ FileStatus fs = getFileStatus(f); // will throw if the file doesn't exist
+ if (fs.isDirectory()) {
+ throw new FileNotFoundException("'" + f + "' is a directory");
+ }
+ LOG.info("Opening '" + f + "' for reading");
+ Path absolutePath = makeAbsolute(f);
+ String key = pathToKey(absolutePath);
+ return new FSDataInputStream(new BufferedFSInputStream(
+ new NativeS3FsInputStream(store, statistics, store.retrieve(key), key), bufferSize));
+ }
+
+ // rename() and delete() use this method to ensure that the parent directory
+ // of the source does not vanish.
+ private void createParent(Path path) throws IOException {
+ Path parent = path.getParent();
+ if (parent != null) {
+ String key = pathToKey(makeAbsolute(parent));
+ if (key.length() > 0) {
+ store.storeEmptyFile(key + FOLDER_SUFFIX);
+ }
+ }
+ }
+
+
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+
+ String srcKey = pathToKey(makeAbsolute(src));
+ final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
+
+ if (srcKey.length() == 0) {
+ // Cannot rename root of file system
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble +
+ "returning false as cannot rename the root of a filesystem");
+ }
+ return false;
+ }
+
+ //get status of source
+ boolean srcIsFile;
+ try {
+ srcIsFile = getFileStatus(src).isFile();
+ } catch (FileNotFoundException e) {
+ //bail out fast if the source does not exist
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble + "returning false as src does not exist");
+ }
+ return false;
+ }
+ // Figure out the final destination
+ String dstKey = pathToKey(makeAbsolute(dst));
+
+ try {
+ boolean dstIsFile = getFileStatus(dst).isFile();
+ if (dstIsFile) {
+ //destination is a file.
+ //you can't copy a file or a directory onto an existing file
+ //except for the special case of dest==src, which is a no-op
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble +
+ "returning without rename as dst is an already existing file");
+ }
+ //exit, returning true iff the rename is onto self
+ return srcKey.equals(dstKey);
+ } else {
+ //destination exists and is a directory
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble + "using dst as output directory");
+ }
+ //destination goes under the dst path, with the name of the
+ //source entry
+ dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName())));
+ }
+ } catch (FileNotFoundException e) {
+ //destination does not exist => the source file or directory
+ //is copied over with the name of the destination
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble + "using dst as output destination");
+ }
+ try {
+ if (getFileStatus(dst.getParent()).isFile()) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble +
+ "returning false as dst parent exists and is a file");
+ }
+ return false;
+ }
+ } catch (FileNotFoundException ex) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble +
+ "returning false as dst parent does not exist");
+ }
+ return false;
+ }
+ }
+
+ //rename to self behavior follows Posix rules and is different
+ //for directories and files -the return code is driven by src type
+ if (srcKey.equals(dstKey)) {
+ //fully resolved destination key matches source: fail
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble + "renamingToSelf; returning true");
+ }
+ return true;
+ }
+ if (srcIsFile) {
+ //source is a file; COPY then DELETE
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble +
+ "src is file, so doing copy then delete in S3");
+ }
+ store.copy(srcKey, dstKey);
+ store.delete(srcKey);
+ } else {
+ //src is a directory
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble + "src is directory, so copying contents");
+ }
+ //Verify dest is not a child of the parent
+ if (dstKey.startsWith(srcKey + "/")) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ debugPreamble + "cannot rename a directory to a subdirectory of self");
+ }
+ return false;
+ }
+ //create the subdir under the destination
+ store.storeEmptyFile(dstKey + FOLDER_SUFFIX);
+
+ List<String> keysToDelete = new ArrayList<String>();
+ String priorLastKey = null;
+ do {
+ PartialListing listing = store.list(srcKey, S3_MAX_LISTING_LENGTH, priorLastKey, true);
+ for (FileMetadata file : listing.getFiles()) {
+ keysToDelete.add(file.getKey());
+ store.copy(file.getKey(), dstKey + file.getKey().substring(srcKey.length()));
+ }
+ priorLastKey = listing.getPriorLastKey();
+ } while (priorLastKey != null);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble +
+ "all files in src copied, now removing src files");
+ }
+ for (String key: keysToDelete) {
+ store.delete(key);
+ }
+
+ try {
+ store.delete(srcKey + FOLDER_SUFFIX);
+ } catch (FileNotFoundException e) {
+ //this is fine, we don't require a marker
+ }
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble + "done");
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public long getDefaultBlockSize() {
+ return getConf().getLong("fs.s3n.block.size", 64 * 1024 * 1024);
+ }
+
+ /**
+ * Set the working directory to the given directory.
+ */
+ @Override
+ public void setWorkingDirectory(Path newDir) {
+ workingDir = newDir;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ @Override
+ public String getCanonicalServiceName() {
+ // Does not support Token
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/PartialListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/PartialListing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/PartialListing.java
new file mode 100644
index 0000000..8290092
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/PartialListing.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * <p>
+ * Holds information on a directory listing for a
+ * {@link NativeFileSystemStore}.
+ * This includes the {@link FileMetadata files} and directories
+ * (their names) contained in a directory.
+ * </p>
+ * <p>
+ * This listing may be returned in chunks, so a <code>priorLastKey</code>
+ * is provided so that the next chunk may be requested.
+ * </p>
+ * @see NativeFileSystemStore#list(String, int, String)
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class PartialListing {
+
+ private final String priorLastKey;
+ private final FileMetadata[] files;
+ private final String[] commonPrefixes;
+
+ public PartialListing(String priorLastKey, FileMetadata[] files,
+ String[] commonPrefixes) {
+ this.priorLastKey = priorLastKey;
+ this.files = files;
+ this.commonPrefixes = commonPrefixes;
+ }
+
+ public FileMetadata[] getFiles() {
+ return files;
+ }
+
+ public String[] getCommonPrefixes() {
+ return commonPrefixes;
+ }
+
+ public String getPriorLastKey() {
+ return priorLastKey;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3NativeFileSystemConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3NativeFileSystemConfigKeys.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3NativeFileSystemConfigKeys.java
new file mode 100644
index 0000000..75884fa
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3NativeFileSystemConfigKeys.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+
+/**
+ * This class contains constants for configuration keys used
+ * in the s3 file system.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class S3NativeFileSystemConfigKeys extends CommonConfigurationKeys {
+ public static final String S3_NATIVE_BLOCK_SIZE_KEY = "s3native.blocksize";
+ public static final long S3_NATIVE_BLOCK_SIZE_DEFAULT = 64*1024*1024;
+ public static final String S3_NATIVE_REPLICATION_KEY = "s3native.replication";
+ public static final short S3_NATIVE_REPLICATION_DEFAULT = 1;
+ public static final String S3_NATIVE_STREAM_BUFFER_SIZE_KEY =
+ "s3native.stream-buffer-size";
+ public static final int S3_NATIVE_STREAM_BUFFER_SIZE_DEFAULT = 4096;
+ public static final String S3_NATIVE_BYTES_PER_CHECKSUM_KEY =
+ "s3native.bytes-per-checksum";
+ public static final int S3_NATIVE_BYTES_PER_CHECKSUM_DEFAULT = 512;
+ public static final String S3_NATIVE_CLIENT_WRITE_PACKET_SIZE_KEY =
+ "s3native.client-write-packet-size";
+ public static final int S3_NATIVE_CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024;
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/package.html
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/package.html b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/package.html
new file mode 100644
index 0000000..24b9b1d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/package.html
@@ -0,0 +1,32 @@
+<html>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<body>
+
+<p>
+A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem} for reading and writing files on
+<a href="http://aws.amazon.com/s3">Amazon S3</a>.
+Unlike {@link org.apache.hadoop.fs.s3.S3FileSystem}, which is block-based,
+this implementation stores
+files on S3 in their native form for interoperability with other S3 tools.
+</p>
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
new file mode 100644
index 0000000..3cd1d6b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.fs.s3.S3FileSystem
+org.apache.hadoop.fs.s3native.NativeS3FileSystem
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
new file mode 100644
index 0000000..ace6444
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
+
+/**
+ * The contract of S3N: only enabled if the test bucket is provided
+ */
+public class NativeS3Contract extends AbstractBondedFSContract {
+
+ public static final String CONTRACT_XML = "contract/s3n.xml";
+
+
+ public NativeS3Contract(Configuration conf) {
+ super(conf);
+ //insert the base features
+ addConfResource(CONTRACT_XML);
+ }
+
+ @Override
+ public String getScheme() {
+ return "s3n";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java
new file mode 100644
index 0000000..e44e2b1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+public class TestS3NContractCreate extends AbstractContractCreateTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+
+ @Override
+ public void testOverwriteEmptyDirectory() throws Throwable {
+ ContractTestUtils.skip(
+ "blobstores can't distinguish empty directories from files");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java
new file mode 100644
index 0000000..1b79d27
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3NContractDelete extends AbstractContractDeleteTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java
new file mode 100644
index 0000000..527a31d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test dir operations on S3
+ */
+public class TestS3NContractMkdir extends AbstractContractMkdirTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java
new file mode 100644
index 0000000..2186f28
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3NContractOpen extends AbstractContractOpenTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java
new file mode 100644
index 0000000..d673416
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3NContractRename extends AbstractContractRenameTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java
new file mode 100644
index 0000000..94f8483
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * root dir operations against an S3 bucket
+ */
+public class TestS3NContractRootDir extends
+ AbstractContractRootDirectoryTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java
new file mode 100644
index 0000000..6d04fff
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3NContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
new file mode 100644
index 0000000..2d43c8b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3.INode.FileType;
+
+/**
+ * A stub implementation of {@link FileSystemStore} for testing
+ * {@link S3FileSystem} without actually connecting to S3.
+ */
+public class InMemoryFileSystemStore implements FileSystemStore {
+
+ private Configuration conf;
+ private SortedMap<Path, INode> inodes = new TreeMap<Path, INode>();
+ private Map<Long, byte[]> blocks = new HashMap<Long, byte[]>();
+
+ @Override
+ public void initialize(URI uri, Configuration conf) {
+ this.conf = conf;
+ inodes.put(new Path("/"), INode.DIRECTORY_INODE);
+ }
+
+ @Override
+ public String getVersion() throws IOException {
+ return "0";
+ }
+
+ @Override
+ public void deleteINode(Path path) throws IOException {
+ inodes.remove(normalize(path));
+ }
+
+ @Override
+ public void deleteBlock(Block block) throws IOException {
+ blocks.remove(block.getId());
+ }
+
+ @Override
+ public boolean inodeExists(Path path) throws IOException {
+ return inodes.containsKey(normalize(path));
+ }
+
+ @Override
+ public boolean blockExists(long blockId) throws IOException {
+ return blocks.containsKey(blockId);
+ }
+
+ @Override
+ public INode retrieveINode(Path path) throws IOException {
+ return inodes.get(normalize(path));
+ }
+
+ @Override
+ public File retrieveBlock(Block block, long byteRangeStart) throws IOException {
+ byte[] data = blocks.get(block.getId());
+ File file = createTempFile();
+ BufferedOutputStream out = null;
+ try {
+ out = new BufferedOutputStream(new FileOutputStream(file));
+ out.write(data, (int) byteRangeStart, data.length - (int) byteRangeStart);
+ } finally {
+ if (out != null) {
+ out.close();
+ }
+ }
+ return file;
+ }
+
+ private File createTempFile() throws IOException {
+ File dir = new File(conf.get("fs.s3.buffer.dir"));
+ if (!dir.exists() && !dir.mkdirs()) {
+ throw new IOException("Cannot create S3 buffer directory: " + dir);
+ }
+ File result = File.createTempFile("test-", ".tmp", dir);
+ result.deleteOnExit();
+ return result;
+ }
+
+ @Override
+ public Set<Path> listSubPaths(Path path) throws IOException {
+ Path normalizedPath = normalize(path);
+ // This is inefficient but more than adequate for testing purposes.
+ Set<Path> subPaths = new LinkedHashSet<Path>();
+ for (Path p : inodes.tailMap(normalizedPath).keySet()) {
+ if (normalizedPath.equals(p.getParent())) {
+ subPaths.add(p);
+ }
+ }
+ return subPaths;
+ }
+
+ @Override
+ public Set<Path> listDeepSubPaths(Path path) throws IOException {
+ Path normalizedPath = normalize(path);
+ String pathString = normalizedPath.toUri().getPath();
+ if (!pathString.endsWith("/")) {
+ pathString += "/";
+ }
+ // This is inefficient but more than adequate for testing purposes.
+ Set<Path> subPaths = new LinkedHashSet<Path>();
+ for (Path p : inodes.tailMap(normalizedPath).keySet()) {
+ if (p.toUri().getPath().startsWith(pathString)) {
+ subPaths.add(p);
+ }
+ }
+ return subPaths;
+ }
+
+ @Override
+ public void storeINode(Path path, INode inode) throws IOException {
+ inodes.put(normalize(path), inode);
+ }
+
+ @Override
+ public void storeBlock(Block block, File file) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ byte[] buf = new byte[8192];
+ int numRead;
+ BufferedInputStream in = null;
+ try {
+ in = new BufferedInputStream(new FileInputStream(file));
+ while ((numRead = in.read(buf)) >= 0) {
+ out.write(buf, 0, numRead);
+ }
+ } finally {
+ if (in != null) {
+ in.close();
+ }
+ }
+ blocks.put(block.getId(), out.toByteArray());
+ }
+
+ private Path normalize(Path path) {
+ if (!path.isAbsolute()) {
+ throw new IllegalArgumentException("Path must be absolute: " + path);
+ }
+ return new Path(path.toUri().getPath());
+ }
+
+ @Override
+ public void purge() throws IOException {
+ inodes.clear();
+ blocks.clear();
+ }
+
+ @Override
+ public void dump() throws IOException {
+ StringBuilder sb = new StringBuilder(getClass().getSimpleName());
+ sb.append(", \n");
+ for (Map.Entry<Path, INode> entry : inodes.entrySet()) {
+ sb.append(entry.getKey()).append("\n");
+ INode inode = entry.getValue();
+ sb.append("\t").append(inode.getFileType()).append("\n");
+ if (inode.getFileType() == FileType.DIRECTORY) {
+ continue;
+ }
+ for (int j = 0; j < inode.getBlocks().length; j++) {
+ sb.append("\t").append(inode.getBlocks()[j]).append("\n");
+ }
+ }
+ System.out.println(sb);
+
+ System.out.println(inodes.keySet());
+ System.out.println(blocks.keySet());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/Jets3tS3FileSystemContractTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/Jets3tS3FileSystemContractTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/Jets3tS3FileSystemContractTest.java
new file mode 100644
index 0000000..53b3c03
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/Jets3tS3FileSystemContractTest.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.IOException;
+
+public class Jets3tS3FileSystemContractTest
+ extends S3FileSystemContractBaseTest {
+
+ @Override
+ FileSystemStore getFileSystemStore() throws IOException {
+ return new Jets3tFileSystemStore();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java
new file mode 100644
index 0000000..28b0507
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+
+public abstract class S3FileSystemContractBaseTest
+ extends FileSystemContractBaseTest {
+
+ private FileSystemStore store;
+
+ abstract FileSystemStore getFileSystemStore() throws IOException;
+
+ @Override
+ protected void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ store = getFileSystemStore();
+ fs = new S3FileSystem(store);
+ fs.initialize(URI.create(conf.get("test.fs.s3.name")), conf);
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ store.purge();
+ super.tearDown();
+ }
+
+ public void testCanonicalName() throws Exception {
+ assertNull("s3 doesn't support security token and shouldn't have canonical name",
+ fs.getCanonicalServiceName());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/S3InMemoryFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/S3InMemoryFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/S3InMemoryFileSystem.java
new file mode 100644
index 0000000..a4e9770
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/S3InMemoryFileSystem.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import org.apache.hadoop.fs.s3.S3FileSystem;
+import org.apache.hadoop.fs.s3.InMemoryFileSystemStore;
+
+/**
+ * A helper implementation of {@link S3FileSystem}
+ * without actually connecting to S3 for unit testing.
+ */
+public class S3InMemoryFileSystem extends S3FileSystem {
+ public S3InMemoryFileSystem() {
+ super(new InMemoryFileSystemStore());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestINode.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestINode.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestINode.java
new file mode 100644
index 0000000..086a43e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestINode.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.fs.s3.INode.FileType;
+
+public class TestINode extends TestCase {
+
+ public void testSerializeFileWithSingleBlock() throws IOException {
+ Block[] blocks = { new Block(849282477840258181L, 128L) };
+ INode inode = new INode(FileType.FILE, blocks);
+
+ assertEquals("Length", 1L + 4 + 16, inode.getSerializedLength());
+ InputStream in = inode.serialize();
+
+ INode deserialized = INode.deserialize(in);
+
+ assertEquals("FileType", inode.getFileType(), deserialized.getFileType());
+ Block[] deserializedBlocks = deserialized.getBlocks();
+ assertEquals("Length", 1, deserializedBlocks.length);
+ assertEquals("Id", blocks[0].getId(), deserializedBlocks[0].getId());
+ assertEquals("Length", blocks[0].getLength(), deserializedBlocks[0]
+ .getLength());
+
+ }
+
+ public void testSerializeDirectory() throws IOException {
+ INode inode = INode.DIRECTORY_INODE;
+ assertEquals("Length", 1L, inode.getSerializedLength());
+ InputStream in = inode.serialize();
+ INode deserialized = INode.deserialize(in);
+ assertSame(INode.DIRECTORY_INODE, deserialized);
+ }
+
+ public void testDeserializeNull() throws IOException {
+ assertNull(INode.deserialize(null));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestInMemoryS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestInMemoryS3FileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestInMemoryS3FileSystemContract.java
new file mode 100644
index 0000000..5d66cf1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestInMemoryS3FileSystemContract.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.IOException;
+
+public class TestInMemoryS3FileSystemContract
+ extends S3FileSystemContractBaseTest {
+
+ @Override
+ FileSystemStore getFileSystemStore() throws IOException {
+ return new InMemoryFileSystemStore();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestS3Credentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestS3Credentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestS3Credentials.java
new file mode 100644
index 0000000..bcbf0dc
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestS3Credentials.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3;
+
+import java.net.URI;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class TestS3Credentials extends TestCase {
+ public void testInvalidHostnameWithUnderscores() throws Exception {
+ S3Credentials s3Credentials = new S3Credentials();
+ try {
+ s3Credentials.initialize(new URI("s3://a:b@c_d"), new Configuration());
+ fail("Should throw IllegalArgumentException");
+ } catch (IllegalArgumentException e) {
+ assertEquals("Invalid hostname in URI s3://a:b@c_d", e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestS3FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestS3FileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestS3FileSystem.java
new file mode 100644
index 0000000..f21989c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestS3FileSystem.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.IOException;
+import java.net.URI;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class TestS3FileSystem extends TestCase {
+
+ public void testInitialization() throws IOException {
+ initializationTest("s3://a:b@c", "s3://a:b@c");
+ initializationTest("s3://a:b@c/", "s3://a:b@c");
+ initializationTest("s3://a:b@c/path", "s3://a:b@c");
+ initializationTest("s3://a@c", "s3://a@c");
+ initializationTest("s3://a@c/", "s3://a@c");
+ initializationTest("s3://a@c/path", "s3://a@c");
+ initializationTest("s3://c", "s3://c");
+ initializationTest("s3://c/", "s3://c");
+ initializationTest("s3://c/path", "s3://c");
+ }
+
+ private void initializationTest(String initializationUri, String expectedUri)
+ throws IOException {
+
+ S3FileSystem fs = new S3FileSystem(new InMemoryFileSystemStore());
+ fs.initialize(URI.create(initializationUri), new Configuration());
+ assertEquals(URI.create(expectedUri), fs.getUri());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestS3InMemoryFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestS3InMemoryFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestS3InMemoryFileSystem.java
new file mode 100644
index 0000000..fbdcd68
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3/TestS3InMemoryFileSystem.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URI;
+import junit.framework.TestCase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+
+public class TestS3InMemoryFileSystem extends TestCase {
+
+ private static final String TEST_PATH = "s3://test/data.txt";
+
+ private static final String TEST_DATA = "Sample data for testing.";
+
+ private S3InMemoryFileSystem fs;
+
+ @Override
+ public void setUp() throws IOException {
+ fs = new S3InMemoryFileSystem();
+ fs.initialize(URI.create("s3://test/"), new Configuration());
+ }
+
+ public void testBasicReadWriteIO() throws IOException {
+ FSDataOutputStream writeStream = fs.create(new Path(TEST_PATH));
+ writeStream.write(TEST_DATA.getBytes());
+ writeStream.flush();
+ writeStream.close();
+
+ FSDataInputStream readStream = fs.open(new Path(TEST_PATH));
+ BufferedReader br = new BufferedReader(new InputStreamReader(readStream));
+ String line = "";
+ StringBuffer stringBuffer = new StringBuffer();
+ while ((line = br.readLine()) != null) {
+ stringBuffer.append(line);
+ }
+ br.close();
+
+ assert(TEST_DATA.equals(stringBuffer.toString()));
+ }
+
+ @Override
+ public void tearDown() throws IOException {
+ fs.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
new file mode 100644
index 0000000..ac572aa
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/InMemoryNativeFileSystemStore.java
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import static org.apache.hadoop.fs.s3native.NativeS3FileSystem.PATH_DELIMITER;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Time;
+
+/**
+ * <p>
+ * A stub implementation of {@link NativeFileSystemStore} for testing
+ * {@link NativeS3FileSystem} without actually connecting to S3.
+ * </p>
+ */
+public class InMemoryNativeFileSystemStore implements NativeFileSystemStore {
+
+ private Configuration conf;
+
+ private SortedMap<String, FileMetadata> metadataMap =
+ new TreeMap<String, FileMetadata>();
+ private SortedMap<String, byte[]> dataMap = new TreeMap<String, byte[]>();
+
+ @Override
+ public void initialize(URI uri, Configuration conf) throws IOException {
+ this.conf = conf;
+ }
+
+ @Override
+ public void storeEmptyFile(String key) throws IOException {
+ metadataMap.put(key, new FileMetadata(key, 0, Time.now()));
+ dataMap.put(key, new byte[0]);
+ }
+
+ @Override
+ public void storeFile(String key, File file, byte[] md5Hash)
+ throws IOException {
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ byte[] buf = new byte[8192];
+ int numRead;
+ BufferedInputStream in = null;
+ try {
+ in = new BufferedInputStream(new FileInputStream(file));
+ while ((numRead = in.read(buf)) >= 0) {
+ out.write(buf, 0, numRead);
+ }
+ } finally {
+ if (in != null) {
+ in.close();
+ }
+ }
+ metadataMap.put(key,
+ new FileMetadata(key, file.length(), Time.now()));
+ dataMap.put(key, out.toByteArray());
+ }
+
+ @Override
+ public InputStream retrieve(String key) throws IOException {
+ return retrieve(key, 0);
+ }
+
+ @Override
+ public InputStream retrieve(String key, long byteRangeStart)
+ throws IOException {
+
+ byte[] data = dataMap.get(key);
+ File file = createTempFile();
+ BufferedOutputStream out = null;
+ try {
+ out = new BufferedOutputStream(new FileOutputStream(file));
+ out.write(data, (int) byteRangeStart,
+ data.length - (int) byteRangeStart);
+ } finally {
+ if (out != null) {
+ out.close();
+ }
+ }
+ return new FileInputStream(file);
+ }
+
+ private File createTempFile() throws IOException {
+ File dir = new File(conf.get("fs.s3.buffer.dir"));
+ if (!dir.exists() && !dir.mkdirs()) {
+ throw new IOException("Cannot create S3 buffer directory: " + dir);
+ }
+ File result = File.createTempFile("test-", ".tmp", dir);
+ result.deleteOnExit();
+ return result;
+ }
+
+ @Override
+ public FileMetadata retrieveMetadata(String key) throws IOException {
+ return metadataMap.get(key);
+ }
+
+ @Override
+ public PartialListing list(String prefix, int maxListingLength)
+ throws IOException {
+ return list(prefix, maxListingLength, null, false);
+ }
+
+ @Override
+ public PartialListing list(String prefix, int maxListingLength,
+ String priorLastKey, boolean recursive) throws IOException {
+
+ return list(prefix, recursive ? null : PATH_DELIMITER, maxListingLength, priorLastKey);
+ }
+
+ private PartialListing list(String prefix, String delimiter,
+ int maxListingLength, String priorLastKey) throws IOException {
+
+ if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) {
+ prefix += PATH_DELIMITER;
+ }
+
+ List<FileMetadata> metadata = new ArrayList<FileMetadata>();
+ SortedSet<String> commonPrefixes = new TreeSet<String>();
+ for (String key : dataMap.keySet()) {
+ if (key.startsWith(prefix)) {
+ if (delimiter == null) {
+ metadata.add(retrieveMetadata(key));
+ } else {
+ int delimIndex = key.indexOf(delimiter, prefix.length());
+ if (delimIndex == -1) {
+ metadata.add(retrieveMetadata(key));
+ } else {
+ String commonPrefix = key.substring(0, delimIndex);
+ commonPrefixes.add(commonPrefix);
+ }
+ }
+ }
+ if (metadata.size() + commonPrefixes.size() == maxListingLength) {
+ new PartialListing(key, metadata.toArray(new FileMetadata[0]),
+ commonPrefixes.toArray(new String[0]));
+ }
+ }
+ return new PartialListing(null, metadata.toArray(new FileMetadata[0]),
+ commonPrefixes.toArray(new String[0]));
+ }
+
+ @Override
+ public void delete(String key) throws IOException {
+ metadataMap.remove(key);
+ dataMap.remove(key);
+ }
+
+ @Override
+ public void copy(String srcKey, String dstKey) throws IOException {
+ metadataMap.put(dstKey, metadataMap.get(srcKey));
+ dataMap.put(dstKey, dataMap.get(srcKey));
+ }
+
+ @Override
+ public void purge(String prefix) throws IOException {
+ Iterator<Entry<String, FileMetadata>> i =
+ metadataMap.entrySet().iterator();
+ while (i.hasNext()) {
+ Entry<String, FileMetadata> entry = i.next();
+ if (entry.getKey().startsWith(prefix)) {
+ dataMap.remove(entry.getKey());
+ i.remove();
+ }
+ }
+ }
+
+ @Override
+ public void dump() throws IOException {
+ System.out.println(metadataMap.values());
+ System.out.println(dataMap.keySet());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/Jets3tNativeS3FileSystemContractTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/Jets3tNativeS3FileSystemContractTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/Jets3tNativeS3FileSystemContractTest.java
new file mode 100644
index 0000000..6516c83
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/Jets3tNativeS3FileSystemContractTest.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.IOException;
+
+public class Jets3tNativeS3FileSystemContractTest
+ extends NativeS3FileSystemContractBaseTest {
+
+ @Override
+ NativeFileSystemStore getNativeFileSystemStore() throws IOException {
+ return new Jets3tNativeFileSystemStore();
+ }
+}
[15/20] HADOOP-11074. Move s3-related FS connector code to
hadoop-aws. (David S. Wang via Colin Patrick McCabe)
Posted by sz...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
deleted file mode 100644
index a10d6f2..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
+++ /dev/null
@@ -1,483 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import static org.apache.hadoop.fs.s3native.NativeS3FileSystem.PATH_DELIMITER;
-
-import java.io.BufferedInputStream;
-import java.io.ByteArrayInputStream;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.s3.S3Credentials;
-import org.apache.hadoop.fs.s3.S3Exception;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.security.AccessControlException;
-import org.jets3t.service.S3Service;
-import org.jets3t.service.S3ServiceException;
-import org.jets3t.service.ServiceException;
-import org.jets3t.service.StorageObjectsChunk;
-import org.jets3t.service.impl.rest.HttpException;
-import org.jets3t.service.impl.rest.httpclient.RestS3Service;
-import org.jets3t.service.model.MultipartPart;
-import org.jets3t.service.model.MultipartUpload;
-import org.jets3t.service.model.S3Bucket;
-import org.jets3t.service.model.S3Object;
-import org.jets3t.service.model.StorageObject;
-import org.jets3t.service.security.AWSCredentials;
-import org.jets3t.service.utils.MultipartUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-class Jets3tNativeFileSystemStore implements NativeFileSystemStore {
-
- private S3Service s3Service;
- private S3Bucket bucket;
-
- private long multipartBlockSize;
- private boolean multipartEnabled;
- private long multipartCopyBlockSize;
- static final long MAX_PART_SIZE = (long)5 * 1024 * 1024 * 1024;
-
- private String serverSideEncryptionAlgorithm;
-
- public static final Logger LOG =
- LoggerFactory.getLogger(Jets3tNativeFileSystemStore.class);
-
- @Override
- public void initialize(URI uri, Configuration conf) throws IOException {
- S3Credentials s3Credentials = new S3Credentials();
- s3Credentials.initialize(uri, conf);
- try {
- AWSCredentials awsCredentials =
- new AWSCredentials(s3Credentials.getAccessKey(),
- s3Credentials.getSecretAccessKey());
- this.s3Service = new RestS3Service(awsCredentials);
- } catch (S3ServiceException e) {
- handleException(e);
- }
- multipartEnabled =
- conf.getBoolean("fs.s3n.multipart.uploads.enabled", false);
- multipartBlockSize = Math.min(
- conf.getLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024),
- MAX_PART_SIZE);
- multipartCopyBlockSize = Math.min(
- conf.getLong("fs.s3n.multipart.copy.block.size", MAX_PART_SIZE),
- MAX_PART_SIZE);
- serverSideEncryptionAlgorithm = conf.get("fs.s3n.server-side-encryption-algorithm");
-
- bucket = new S3Bucket(uri.getHost());
- }
-
- @Override
- public void storeFile(String key, File file, byte[] md5Hash)
- throws IOException {
-
- if (multipartEnabled && file.length() >= multipartBlockSize) {
- storeLargeFile(key, file, md5Hash);
- return;
- }
-
- BufferedInputStream in = null;
- try {
- in = new BufferedInputStream(new FileInputStream(file));
- S3Object object = new S3Object(key);
- object.setDataInputStream(in);
- object.setContentType("binary/octet-stream");
- object.setContentLength(file.length());
- object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
- if (md5Hash != null) {
- object.setMd5Hash(md5Hash);
- }
- s3Service.putObject(bucket, object);
- } catch (ServiceException e) {
- handleException(e, key);
- } finally {
- IOUtils.closeStream(in);
- }
- }
-
- public void storeLargeFile(String key, File file, byte[] md5Hash)
- throws IOException {
- S3Object object = new S3Object(key);
- object.setDataInputFile(file);
- object.setContentType("binary/octet-stream");
- object.setContentLength(file.length());
- object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
- if (md5Hash != null) {
- object.setMd5Hash(md5Hash);
- }
-
- List<StorageObject> objectsToUploadAsMultipart =
- new ArrayList<StorageObject>();
- objectsToUploadAsMultipart.add(object);
- MultipartUtils mpUtils = new MultipartUtils(multipartBlockSize);
-
- try {
- mpUtils.uploadObjects(bucket.getName(), s3Service,
- objectsToUploadAsMultipart, null);
- } catch (Exception e) {
- handleException(e, key);
- }
- }
-
- @Override
- public void storeEmptyFile(String key) throws IOException {
- try {
- S3Object object = new S3Object(key);
- object.setDataInputStream(new ByteArrayInputStream(new byte[0]));
- object.setContentType("binary/octet-stream");
- object.setContentLength(0);
- object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
- s3Service.putObject(bucket, object);
- } catch (ServiceException e) {
- handleException(e, key);
- }
- }
-
- @Override
- public FileMetadata retrieveMetadata(String key) throws IOException {
- StorageObject object = null;
- try {
- LOG.debug("Getting metadata for key: {} from bucket: {}",
- key, bucket.getName());
- object = s3Service.getObjectDetails(bucket.getName(), key);
- return new FileMetadata(key, object.getContentLength(),
- object.getLastModifiedDate().getTime());
-
- } catch (ServiceException e) {
- try {
- // process
- handleException(e, key);
- return null;
- } catch (FileNotFoundException fnfe) {
- // and downgrade missing files
- return null;
- }
- } finally {
- if (object != null) {
- object.closeDataInputStream();
- }
- }
- }
-
- /**
- * @param key
- * The key is the object name that is being retrieved from the S3 bucket
- * @return
- * This method returns null if the key is not found
- * @throws IOException
- */
-
- @Override
- public InputStream retrieve(String key) throws IOException {
- try {
- LOG.debug("Getting key: {} from bucket: {}",
- key, bucket.getName());
- S3Object object = s3Service.getObject(bucket.getName(), key);
- return object.getDataInputStream();
- } catch (ServiceException e) {
- handleException(e, key);
- return null; //return null if key not found
- }
- }
-
- /**
- *
- * @param key
- * The key is the object name that is being retrieved from the S3 bucket
- * @return
- * This method returns null if the key is not found
- * @throws IOException
- */
-
- @Override
- public InputStream retrieve(String key, long byteRangeStart)
- throws IOException {
- try {
- LOG.debug("Getting key: {} from bucket: {} with byteRangeStart: {}",
- key, bucket.getName(), byteRangeStart);
- S3Object object = s3Service.getObject(bucket, key, null, null, null,
- null, byteRangeStart, null);
- return object.getDataInputStream();
- } catch (ServiceException e) {
- handleException(e, key);
- return null;
- }
- }
-
- @Override
- public PartialListing list(String prefix, int maxListingLength)
- throws IOException {
- return list(prefix, maxListingLength, null, false);
- }
-
- @Override
- public PartialListing list(String prefix, int maxListingLength, String priorLastKey,
- boolean recurse) throws IOException {
-
- return list(prefix, recurse ? null : PATH_DELIMITER, maxListingLength, priorLastKey);
- }
-
- /**
- * list objects
- * @param prefix prefix
- * @param delimiter delimiter
- * @param maxListingLength max no. of entries
- * @param priorLastKey last key in any previous search
- * @return a list of matches
- * @throws IOException on any reported failure
- */
-
- private PartialListing list(String prefix, String delimiter,
- int maxListingLength, String priorLastKey) throws IOException {
- try {
- if (!prefix.isEmpty() && !prefix.endsWith(PATH_DELIMITER)) {
- prefix += PATH_DELIMITER;
- }
- StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
- prefix, delimiter, maxListingLength, priorLastKey);
-
- FileMetadata[] fileMetadata =
- new FileMetadata[chunk.getObjects().length];
- for (int i = 0; i < fileMetadata.length; i++) {
- StorageObject object = chunk.getObjects()[i];
- fileMetadata[i] = new FileMetadata(object.getKey(),
- object.getContentLength(), object.getLastModifiedDate().getTime());
- }
- return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
- chunk.getCommonPrefixes());
- } catch (ServiceException e) {
- handleException(e, prefix);
- return null; // never returned - keep compiler happy
- }
- }
-
- @Override
- public void delete(String key) throws IOException {
- try {
- LOG.debug("Deleting key: {} from bucket: {}",
- key, bucket.getName());
- s3Service.deleteObject(bucket, key);
- } catch (ServiceException e) {
- handleException(e, key);
- }
- }
-
- public void rename(String srcKey, String dstKey) throws IOException {
- try {
- s3Service.renameObject(bucket.getName(), srcKey, new S3Object(dstKey));
- } catch (ServiceException e) {
- handleException(e, srcKey);
- }
- }
-
- @Override
- public void copy(String srcKey, String dstKey) throws IOException {
- try {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Copying srcKey: " + srcKey + "to dstKey: " + dstKey + "in bucket: " + bucket.getName());
- }
- if (multipartEnabled) {
- S3Object object = s3Service.getObjectDetails(bucket, srcKey, null,
- null, null, null);
- if (multipartCopyBlockSize > 0 &&
- object.getContentLength() > multipartCopyBlockSize) {
- copyLargeFile(object, dstKey);
- return;
- }
- }
-
- S3Object dstObject = new S3Object(dstKey);
- dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
- s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(),
- dstObject, false);
- } catch (ServiceException e) {
- handleException(e, srcKey);
- }
- }
-
- public void copyLargeFile(S3Object srcObject, String dstKey) throws IOException {
- try {
- long partCount = srcObject.getContentLength() / multipartCopyBlockSize +
- (srcObject.getContentLength() % multipartCopyBlockSize > 0 ? 1 : 0);
-
- MultipartUpload multipartUpload = s3Service.multipartStartUpload
- (bucket.getName(), dstKey, srcObject.getMetadataMap());
-
- List<MultipartPart> listedParts = new ArrayList<MultipartPart>();
- for (int i = 0; i < partCount; i++) {
- long byteRangeStart = i * multipartCopyBlockSize;
- long byteLength;
- if (i < partCount - 1) {
- byteLength = multipartCopyBlockSize;
- } else {
- byteLength = srcObject.getContentLength() % multipartCopyBlockSize;
- if (byteLength == 0) {
- byteLength = multipartCopyBlockSize;
- }
- }
-
- MultipartPart copiedPart = s3Service.multipartUploadPartCopy
- (multipartUpload, i + 1, bucket.getName(), srcObject.getKey(),
- null, null, null, null, byteRangeStart,
- byteRangeStart + byteLength - 1, null);
- listedParts.add(copiedPart);
- }
-
- Collections.reverse(listedParts);
- s3Service.multipartCompleteUpload(multipartUpload, listedParts);
- } catch (ServiceException e) {
- handleException(e, srcObject.getKey());
- }
- }
-
- @Override
- public void purge(String prefix) throws IOException {
- String key = "";
- try {
- S3Object[] objects =
- s3Service.listObjects(bucket.getName(), prefix, null);
- for (S3Object object : objects) {
- key = object.getKey();
- s3Service.deleteObject(bucket, key);
- }
- } catch (S3ServiceException e) {
- handleException(e, key);
- }
- }
-
- @Override
- public void dump() throws IOException {
- StringBuilder sb = new StringBuilder("S3 Native Filesystem, ");
- sb.append(bucket.getName()).append("\n");
- try {
- S3Object[] objects = s3Service.listObjects(bucket.getName());
- for (S3Object object : objects) {
- sb.append(object.getKey()).append("\n");
- }
- } catch (S3ServiceException e) {
- handleException(e);
- }
- System.out.println(sb);
- }
-
- /**
- * Handle any service exception by translating it into an IOException
- * @param e exception
- * @throws IOException exception -always
- */
- private void handleException(Exception e) throws IOException {
- throw processException(e, e, "");
- }
- /**
- * Handle any service exception by translating it into an IOException
- * @param e exception
- * @param key key sought from object store
-
- * @throws IOException exception -always
- */
- private void handleException(Exception e, String key) throws IOException {
- throw processException(e, e, key);
- }
-
- /**
- * Handle any service exception by translating it into an IOException
- * @param thrown exception
- * @param original original exception -thrown if no other translation could
- * be made
- * @param key key sought from object store or "" for undefined
- * @return an exception to throw. If isProcessingCause==true this may be null.
- */
- private IOException processException(Throwable thrown, Throwable original,
- String key) {
- IOException result;
- if (thrown.getCause() != null) {
- // recurse down
- result = processException(thrown.getCause(), original, key);
- } else if (thrown instanceof HttpException) {
- // nested HttpException - examine error code and react
- HttpException httpException = (HttpException) thrown;
- String responseMessage = httpException.getResponseMessage();
- int responseCode = httpException.getResponseCode();
- String bucketName = "s3n://" + bucket.getName();
- String text = String.format("%s : %03d : %s",
- bucketName,
- responseCode,
- responseMessage);
- String filename = !key.isEmpty() ? (bucketName + "/" + key) : text;
- IOException ioe;
- switch (responseCode) {
- case 404:
- result = new FileNotFoundException(filename);
- break;
- case 416: // invalid range
- result = new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF
- +": " + filename);
- break;
- case 403: //forbidden
- result = new AccessControlException("Permission denied"
- +": " + filename);
- break;
- default:
- result = new IOException(text);
- }
- result.initCause(thrown);
- } else if (thrown instanceof S3ServiceException) {
- S3ServiceException se = (S3ServiceException) thrown;
- LOG.debug(
- "S3ServiceException: {}: {} : {}",
- se.getS3ErrorCode(), se.getS3ErrorMessage(), se, se);
- if ("InvalidRange".equals(se.getS3ErrorCode())) {
- result = new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
- } else {
- result = new S3Exception(se);
- }
- } else if (thrown instanceof ServiceException) {
- ServiceException se = (ServiceException) thrown;
- LOG.debug("S3ServiceException: {}: {} : {}",
- se.getErrorCode(), se.toString(), se, se);
- result = new S3Exception(se);
- } else if (thrown instanceof IOException) {
- result = (IOException) thrown;
- } else {
- // here there is no exception derived yet.
- // this means no inner cause, and no translation made yet.
- // convert the original to an IOException -rather than just the
- // exception at the base of the tree
- result = new S3Exception(original);
- }
-
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
deleted file mode 100644
index f26cdac..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeFileSystemStore.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * <p>
- * An abstraction for a key-based {@link File} store.
- * </p>
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-interface NativeFileSystemStore {
-
- void initialize(URI uri, Configuration conf) throws IOException;
-
- void storeFile(String key, File file, byte[] md5Hash) throws IOException;
- void storeEmptyFile(String key) throws IOException;
-
- FileMetadata retrieveMetadata(String key) throws IOException;
- InputStream retrieve(String key) throws IOException;
- InputStream retrieve(String key, long byteRangeStart) throws IOException;
-
- PartialListing list(String prefix, int maxListingLength) throws IOException;
- PartialListing list(String prefix, int maxListingLength, String priorLastKey, boolean recursive)
- throws IOException;
-
- void delete(String key) throws IOException;
-
- void copy(String srcKey, String dstKey) throws IOException;
-
- /**
- * Delete all keys with the given prefix. Used for testing.
- * @throws IOException
- */
- void purge(String prefix) throws IOException;
-
- /**
- * Diagnostic method to dump state to the console.
- * @throws IOException
- */
- void dump() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
deleted file mode 100644
index acc5500..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
+++ /dev/null
@@ -1,796 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import java.io.BufferedOutputStream;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.security.DigestOutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BufferedFSInputStream;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalDirAllocator;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.s3.S3Exception;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryProxy;
-import org.apache.hadoop.util.Progressable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * A {@link FileSystem} for reading and writing files stored on
- * <a href="http://aws.amazon.com/s3">Amazon S3</a>.
- * Unlike {@link org.apache.hadoop.fs.s3.S3FileSystem} this implementation
- * stores files on S3 in their
- * native form so they can be read by other S3 tools.
- *
- * A note about directories. S3 of course has no "native" support for them.
- * The idiom we choose then is: for any directory created by this class,
- * we use an empty object "#{dirpath}_$folder$" as a marker.
- * Further, to interoperate with other S3 tools, we also accept the following:
- * - an object "#{dirpath}/' denoting a directory marker
- * - if there exists any objects with the prefix "#{dirpath}/", then the
- * directory is said to exist
- * - if both a file with the name of a directory and a marker for that
- * directory exists, then the *file masks the directory*, and the directory
- * is never returned.
- * </p>
- * @see org.apache.hadoop.fs.s3.S3FileSystem
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class NativeS3FileSystem extends FileSystem {
-
- public static final Logger LOG =
- LoggerFactory.getLogger(NativeS3FileSystem.class);
-
- private static final String FOLDER_SUFFIX = "_$folder$";
- static final String PATH_DELIMITER = Path.SEPARATOR;
- private static final int S3_MAX_LISTING_LENGTH = 1000;
-
- static class NativeS3FsInputStream extends FSInputStream {
-
- private NativeFileSystemStore store;
- private Statistics statistics;
- private InputStream in;
- private final String key;
- private long pos = 0;
-
- public NativeS3FsInputStream(NativeFileSystemStore store, Statistics statistics, InputStream in, String key) {
- Preconditions.checkNotNull(in, "Null input stream");
- this.store = store;
- this.statistics = statistics;
- this.in = in;
- this.key = key;
- }
-
- @Override
- public synchronized int read() throws IOException {
- int result;
- try {
- result = in.read();
- } catch (IOException e) {
- LOG.info("Received IOException while reading '{}', attempting to reopen",
- key);
- LOG.debug("{}", e, e);
- try {
- seek(pos);
- result = in.read();
- } catch (EOFException eof) {
- LOG.debug("EOF on input stream read: {}", eof, eof);
- result = -1;
- }
- }
- if (result != -1) {
- pos++;
- }
- if (statistics != null && result != -1) {
- statistics.incrementBytesRead(1);
- }
- return result;
- }
- @Override
- public synchronized int read(byte[] b, int off, int len)
- throws IOException {
- if (in == null) {
- throw new EOFException("Cannot read closed stream");
- }
- int result = -1;
- try {
- result = in.read(b, off, len);
- } catch (EOFException eof) {
- throw eof;
- } catch (IOException e) {
- LOG.info( "Received IOException while reading '{}'," +
- " attempting to reopen.", key);
- seek(pos);
- result = in.read(b, off, len);
- }
- if (result > 0) {
- pos += result;
- }
- if (statistics != null && result > 0) {
- statistics.incrementBytesRead(result);
- }
- return result;
- }
-
- @Override
- public synchronized void close() throws IOException {
- closeInnerStream();
- }
-
- /**
- * Close the inner stream if not null. Even if an exception
- * is raised during the close, the field is set to null
- * @throws IOException if raised by the close() operation.
- */
- private void closeInnerStream() throws IOException {
- if (in != null) {
- try {
- in.close();
- } finally {
- in = null;
- }
- }
- }
-
- /**
- * Update inner stream with a new stream and position
- * @param newStream new stream -must not be null
- * @param newpos new position
- * @throws IOException IO exception on a failure to close the existing
- * stream.
- */
- private synchronized void updateInnerStream(InputStream newStream, long newpos) throws IOException {
- Preconditions.checkNotNull(newStream, "Null newstream argument");
- closeInnerStream();
- in = newStream;
- this.pos = newpos;
- }
-
- @Override
- public synchronized void seek(long newpos) throws IOException {
- if (newpos < 0) {
- throw new EOFException(
- FSExceptionMessages.NEGATIVE_SEEK);
- }
- if (pos != newpos) {
- // the seek is attempting to move the current position
- LOG.debug("Opening key '{}' for reading at position '{}", key, newpos);
- InputStream newStream = store.retrieve(key, newpos);
- updateInnerStream(newStream, newpos);
- }
- }
-
- @Override
- public synchronized long getPos() throws IOException {
- return pos;
- }
- @Override
- public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
- }
-
- private class NativeS3FsOutputStream extends OutputStream {
-
- private Configuration conf;
- private String key;
- private File backupFile;
- private OutputStream backupStream;
- private MessageDigest digest;
- private boolean closed;
- private LocalDirAllocator lDirAlloc;
-
- public NativeS3FsOutputStream(Configuration conf,
- NativeFileSystemStore store, String key, Progressable progress,
- int bufferSize) throws IOException {
- this.conf = conf;
- this.key = key;
- this.backupFile = newBackupFile();
- LOG.info("OutputStream for key '" + key + "' writing to tempfile '" + this.backupFile + "'");
- try {
- this.digest = MessageDigest.getInstance("MD5");
- this.backupStream = new BufferedOutputStream(new DigestOutputStream(
- new FileOutputStream(backupFile), this.digest));
- } catch (NoSuchAlgorithmException e) {
- LOG.warn("Cannot load MD5 digest algorithm," +
- "skipping message integrity check.", e);
- this.backupStream = new BufferedOutputStream(
- new FileOutputStream(backupFile));
- }
- }
-
- private File newBackupFile() throws IOException {
- if (lDirAlloc == null) {
- lDirAlloc = new LocalDirAllocator("fs.s3.buffer.dir");
- }
- File result = lDirAlloc.createTmpFileForWrite("output-", LocalDirAllocator.SIZE_UNKNOWN, conf);
- result.deleteOnExit();
- return result;
- }
-
- @Override
- public void flush() throws IOException {
- backupStream.flush();
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (closed) {
- return;
- }
-
- backupStream.close();
- LOG.info("OutputStream for key '{}' closed. Now beginning upload", key);
-
- try {
- byte[] md5Hash = digest == null ? null : digest.digest();
- store.storeFile(key, backupFile, md5Hash);
- } finally {
- if (!backupFile.delete()) {
- LOG.warn("Could not delete temporary s3n file: " + backupFile);
- }
- super.close();
- closed = true;
- }
- LOG.info("OutputStream for key '{}' upload complete", key);
- }
-
- @Override
- public void write(int b) throws IOException {
- backupStream.write(b);
- }
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- backupStream.write(b, off, len);
- }
- }
-
- private URI uri;
- private NativeFileSystemStore store;
- private Path workingDir;
-
- public NativeS3FileSystem() {
- // set store in initialize()
- }
-
- public NativeS3FileSystem(NativeFileSystemStore store) {
- this.store = store;
- }
-
- /**
- * Return the protocol scheme for the FileSystem.
- * <p/>
- *
- * @return <code>s3n</code>
- */
- @Override
- public String getScheme() {
- return "s3n";
- }
-
- @Override
- public void initialize(URI uri, Configuration conf) throws IOException {
- super.initialize(uri, conf);
- if (store == null) {
- store = createDefaultStore(conf);
- }
- store.initialize(uri, conf);
- setConf(conf);
- this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
- this.workingDir =
- new Path("/user", System.getProperty("user.name")).makeQualified(this.uri, this.getWorkingDirectory());
- }
-
- private static NativeFileSystemStore createDefaultStore(Configuration conf) {
- NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
-
- RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
- conf.getInt("fs.s3.maxRetries", 4),
- conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
- Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
- new HashMap<Class<? extends Exception>, RetryPolicy>();
- exceptionToPolicyMap.put(IOException.class, basePolicy);
- exceptionToPolicyMap.put(S3Exception.class, basePolicy);
-
- RetryPolicy methodPolicy = RetryPolicies.retryByException(
- RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
- Map<String, RetryPolicy> methodNameToPolicyMap =
- new HashMap<String, RetryPolicy>();
- methodNameToPolicyMap.put("storeFile", methodPolicy);
- methodNameToPolicyMap.put("rename", methodPolicy);
-
- return (NativeFileSystemStore)
- RetryProxy.create(NativeFileSystemStore.class, store,
- methodNameToPolicyMap);
- }
-
- private static String pathToKey(Path path) {
- if (path.toUri().getScheme() != null && path.toUri().getPath().isEmpty()) {
- // allow uris without trailing slash after bucket to refer to root,
- // like s3n://mybucket
- return "";
- }
- if (!path.isAbsolute()) {
- throw new IllegalArgumentException("Path must be absolute: " + path);
- }
- String ret = path.toUri().getPath().substring(1); // remove initial slash
- if (ret.endsWith("/") && (ret.indexOf("/") != ret.length() - 1)) {
- ret = ret.substring(0, ret.length() -1);
- }
- return ret;
- }
-
- private static Path keyToPath(String key) {
- return new Path("/" + key);
- }
-
- private Path makeAbsolute(Path path) {
- if (path.isAbsolute()) {
- return path;
- }
- return new Path(workingDir, path);
- }
-
- /** This optional operation is not yet supported. */
- @Override
- public FSDataOutputStream append(Path f, int bufferSize,
- Progressable progress) throws IOException {
- throw new IOException("Not supported");
- }
-
- @Override
- public FSDataOutputStream create(Path f, FsPermission permission,
- boolean overwrite, int bufferSize, short replication, long blockSize,
- Progressable progress) throws IOException {
-
- if (exists(f) && !overwrite) {
- throw new FileAlreadyExistsException("File already exists: " + f);
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("Creating new file '" + f + "' in S3");
- }
- Path absolutePath = makeAbsolute(f);
- String key = pathToKey(absolutePath);
- return new FSDataOutputStream(new NativeS3FsOutputStream(getConf(), store,
- key, progress, bufferSize), statistics);
- }
-
- @Override
- public boolean delete(Path f, boolean recurse) throws IOException {
- FileStatus status;
- try {
- status = getFileStatus(f);
- } catch (FileNotFoundException e) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Delete called for '" + f +
- "' but file does not exist, so returning false");
- }
- return false;
- }
- Path absolutePath = makeAbsolute(f);
- String key = pathToKey(absolutePath);
- if (status.isDirectory()) {
- if (!recurse && listStatus(f).length > 0) {
- throw new IOException("Can not delete " + f + " as is a not empty directory and recurse option is false");
- }
-
- createParent(f);
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("Deleting directory '" + f + "'");
- }
- String priorLastKey = null;
- do {
- PartialListing listing = store.list(key, S3_MAX_LISTING_LENGTH, priorLastKey, true);
- for (FileMetadata file : listing.getFiles()) {
- store.delete(file.getKey());
- }
- priorLastKey = listing.getPriorLastKey();
- } while (priorLastKey != null);
-
- try {
- store.delete(key + FOLDER_SUFFIX);
- } catch (FileNotFoundException e) {
- //this is fine, we don't require a marker
- }
- } else {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Deleting file '" + f + "'");
- }
- createParent(f);
- store.delete(key);
- }
- return true;
- }
-
- @Override
- public FileStatus getFileStatus(Path f) throws IOException {
- Path absolutePath = makeAbsolute(f);
- String key = pathToKey(absolutePath);
-
- if (key.length() == 0) { // root always exists
- return newDirectory(absolutePath);
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("getFileStatus retrieving metadata for key '" + key + "'");
- }
- FileMetadata meta = store.retrieveMetadata(key);
- if (meta != null) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("getFileStatus returning 'file' for key '" + key + "'");
- }
- return newFile(meta, absolutePath);
- }
- if (store.retrieveMetadata(key + FOLDER_SUFFIX) != null) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("getFileStatus returning 'directory' for key '" + key +
- "' as '" + key + FOLDER_SUFFIX + "' exists");
- }
- return newDirectory(absolutePath);
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("getFileStatus listing key '" + key + "'");
- }
- PartialListing listing = store.list(key, 1);
- if (listing.getFiles().length > 0 ||
- listing.getCommonPrefixes().length > 0) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("getFileStatus returning 'directory' for key '" + key +
- "' as it has contents");
- }
- return newDirectory(absolutePath);
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("getFileStatus could not find key '" + key + "'");
- }
- throw new FileNotFoundException("No such file or directory '" + absolutePath + "'");
- }
-
- @Override
- public URI getUri() {
- return uri;
- }
-
- /**
- * <p>
- * If <code>f</code> is a file, this method will make a single call to S3.
- * If <code>f</code> is a directory, this method will make a maximum of
- * (<i>n</i> / 1000) + 2 calls to S3, where <i>n</i> is the total number of
- * files and directories contained directly in <code>f</code>.
- * </p>
- */
- @Override
- public FileStatus[] listStatus(Path f) throws IOException {
-
- Path absolutePath = makeAbsolute(f);
- String key = pathToKey(absolutePath);
-
- if (key.length() > 0) {
- FileMetadata meta = store.retrieveMetadata(key);
- if (meta != null) {
- return new FileStatus[] { newFile(meta, absolutePath) };
- }
- }
-
- URI pathUri = absolutePath.toUri();
- Set<FileStatus> status = new TreeSet<FileStatus>();
- String priorLastKey = null;
- do {
- PartialListing listing = store.list(key, S3_MAX_LISTING_LENGTH, priorLastKey, false);
- for (FileMetadata fileMetadata : listing.getFiles()) {
- Path subpath = keyToPath(fileMetadata.getKey());
- String relativePath = pathUri.relativize(subpath.toUri()).getPath();
-
- if (fileMetadata.getKey().equals(key + "/")) {
- // this is just the directory we have been asked to list
- }
- else if (relativePath.endsWith(FOLDER_SUFFIX)) {
- status.add(newDirectory(new Path(
- absolutePath,
- relativePath.substring(0, relativePath.indexOf(FOLDER_SUFFIX)))));
- }
- else {
- status.add(newFile(fileMetadata, subpath));
- }
- }
- for (String commonPrefix : listing.getCommonPrefixes()) {
- Path subpath = keyToPath(commonPrefix);
- String relativePath = pathUri.relativize(subpath.toUri()).getPath();
- status.add(newDirectory(new Path(absolutePath, relativePath)));
- }
- priorLastKey = listing.getPriorLastKey();
- } while (priorLastKey != null);
-
- if (status.isEmpty() &&
- key.length() > 0 &&
- store.retrieveMetadata(key + FOLDER_SUFFIX) == null) {
- throw new FileNotFoundException("File " + f + " does not exist.");
- }
-
- return status.toArray(new FileStatus[status.size()]);
- }
-
- private FileStatus newFile(FileMetadata meta, Path path) {
- return new FileStatus(meta.getLength(), false, 1, getDefaultBlockSize(),
- meta.getLastModified(), path.makeQualified(this.getUri(), this.getWorkingDirectory()));
- }
-
- private FileStatus newDirectory(Path path) {
- return new FileStatus(0, true, 1, 0, 0, path.makeQualified(this.getUri(), this.getWorkingDirectory()));
- }
-
- @Override
- public boolean mkdirs(Path f, FsPermission permission) throws IOException {
- Path absolutePath = makeAbsolute(f);
- List<Path> paths = new ArrayList<Path>();
- do {
- paths.add(0, absolutePath);
- absolutePath = absolutePath.getParent();
- } while (absolutePath != null);
-
- boolean result = true;
- for (Path path : paths) {
- result &= mkdir(path);
- }
- return result;
- }
-
- private boolean mkdir(Path f) throws IOException {
- try {
- FileStatus fileStatus = getFileStatus(f);
- if (fileStatus.isFile()) {
- throw new FileAlreadyExistsException(String.format(
- "Can't make directory for path '%s' since it is a file.", f));
-
- }
- } catch (FileNotFoundException e) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Making dir '" + f + "' in S3");
- }
- String key = pathToKey(f) + FOLDER_SUFFIX;
- store.storeEmptyFile(key);
- }
- return true;
- }
-
- @Override
- public FSDataInputStream open(Path f, int bufferSize) throws IOException {
- FileStatus fs = getFileStatus(f); // will throw if the file doesn't exist
- if (fs.isDirectory()) {
- throw new FileNotFoundException("'" + f + "' is a directory");
- }
- LOG.info("Opening '" + f + "' for reading");
- Path absolutePath = makeAbsolute(f);
- String key = pathToKey(absolutePath);
- return new FSDataInputStream(new BufferedFSInputStream(
- new NativeS3FsInputStream(store, statistics, store.retrieve(key), key), bufferSize));
- }
-
- // rename() and delete() use this method to ensure that the parent directory
- // of the source does not vanish.
- private void createParent(Path path) throws IOException {
- Path parent = path.getParent();
- if (parent != null) {
- String key = pathToKey(makeAbsolute(parent));
- if (key.length() > 0) {
- store.storeEmptyFile(key + FOLDER_SUFFIX);
- }
- }
- }
-
-
- @Override
- public boolean rename(Path src, Path dst) throws IOException {
-
- String srcKey = pathToKey(makeAbsolute(src));
- final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
-
- if (srcKey.length() == 0) {
- // Cannot rename root of file system
- if (LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble +
- "returning false as cannot rename the root of a filesystem");
- }
- return false;
- }
-
- //get status of source
- boolean srcIsFile;
- try {
- srcIsFile = getFileStatus(src).isFile();
- } catch (FileNotFoundException e) {
- //bail out fast if the source does not exist
- if (LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble + "returning false as src does not exist");
- }
- return false;
- }
- // Figure out the final destination
- String dstKey = pathToKey(makeAbsolute(dst));
-
- try {
- boolean dstIsFile = getFileStatus(dst).isFile();
- if (dstIsFile) {
- //destination is a file.
- //you can't copy a file or a directory onto an existing file
- //except for the special case of dest==src, which is a no-op
- if(LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble +
- "returning without rename as dst is an already existing file");
- }
- //exit, returning true iff the rename is onto self
- return srcKey.equals(dstKey);
- } else {
- //destination exists and is a directory
- if(LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble + "using dst as output directory");
- }
- //destination goes under the dst path, with the name of the
- //source entry
- dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName())));
- }
- } catch (FileNotFoundException e) {
- //destination does not exist => the source file or directory
- //is copied over with the name of the destination
- if(LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble + "using dst as output destination");
- }
- try {
- if (getFileStatus(dst.getParent()).isFile()) {
- if(LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble +
- "returning false as dst parent exists and is a file");
- }
- return false;
- }
- } catch (FileNotFoundException ex) {
- if(LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble +
- "returning false as dst parent does not exist");
- }
- return false;
- }
- }
-
- //rename to self behavior follows Posix rules and is different
- //for directories and files -the return code is driven by src type
- if (srcKey.equals(dstKey)) {
- //fully resolved destination key matches source: fail
- if (LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble + "renamingToSelf; returning true");
- }
- return true;
- }
- if (srcIsFile) {
- //source is a file; COPY then DELETE
- if(LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble +
- "src is file, so doing copy then delete in S3");
- }
- store.copy(srcKey, dstKey);
- store.delete(srcKey);
- } else {
- //src is a directory
- if(LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble + "src is directory, so copying contents");
- }
- //Verify dest is not a child of the parent
- if (dstKey.startsWith(srcKey + "/")) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(
- debugPreamble + "cannot rename a directory to a subdirectory of self");
- }
- return false;
- }
- //create the subdir under the destination
- store.storeEmptyFile(dstKey + FOLDER_SUFFIX);
-
- List<String> keysToDelete = new ArrayList<String>();
- String priorLastKey = null;
- do {
- PartialListing listing = store.list(srcKey, S3_MAX_LISTING_LENGTH, priorLastKey, true);
- for (FileMetadata file : listing.getFiles()) {
- keysToDelete.add(file.getKey());
- store.copy(file.getKey(), dstKey + file.getKey().substring(srcKey.length()));
- }
- priorLastKey = listing.getPriorLastKey();
- } while (priorLastKey != null);
-
- if(LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble +
- "all files in src copied, now removing src files");
- }
- for (String key: keysToDelete) {
- store.delete(key);
- }
-
- try {
- store.delete(srcKey + FOLDER_SUFFIX);
- } catch (FileNotFoundException e) {
- //this is fine, we don't require a marker
- }
- if(LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble + "done");
- }
- }
-
- return true;
- }
-
- @Override
- public long getDefaultBlockSize() {
- return getConf().getLong("fs.s3n.block.size", 64 * 1024 * 1024);
- }
-
- /**
- * Set the working directory to the given directory.
- */
- @Override
- public void setWorkingDirectory(Path newDir) {
- workingDir = newDir;
- }
-
- @Override
- public Path getWorkingDirectory() {
- return workingDir;
- }
-
- @Override
- public String getCanonicalServiceName() {
- // Does not support Token
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/PartialListing.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/PartialListing.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/PartialListing.java
deleted file mode 100644
index 8290092..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/PartialListing.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * <p>
- * Holds information on a directory listing for a
- * {@link NativeFileSystemStore}.
- * This includes the {@link FileMetadata files} and directories
- * (their names) contained in a directory.
- * </p>
- * <p>
- * This listing may be returned in chunks, so a <code>priorLastKey</code>
- * is provided so that the next chunk may be requested.
- * </p>
- * @see NativeFileSystemStore#list(String, int, String)
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-class PartialListing {
-
- private final String priorLastKey;
- private final FileMetadata[] files;
- private final String[] commonPrefixes;
-
- public PartialListing(String priorLastKey, FileMetadata[] files,
- String[] commonPrefixes) {
- this.priorLastKey = priorLastKey;
- this.files = files;
- this.commonPrefixes = commonPrefixes;
- }
-
- public FileMetadata[] getFiles() {
- return files;
- }
-
- public String[] getCommonPrefixes() {
- return commonPrefixes;
- }
-
- public String getPriorLastKey() {
- return priorLastKey;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/S3NativeFileSystemConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/S3NativeFileSystemConfigKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/S3NativeFileSystemConfigKeys.java
deleted file mode 100644
index 75884fa..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/S3NativeFileSystemConfigKeys.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-
-/**
- * This class contains constants for configuration keys used
- * in the s3 file system.
- *
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class S3NativeFileSystemConfigKeys extends CommonConfigurationKeys {
- public static final String S3_NATIVE_BLOCK_SIZE_KEY = "s3native.blocksize";
- public static final long S3_NATIVE_BLOCK_SIZE_DEFAULT = 64*1024*1024;
- public static final String S3_NATIVE_REPLICATION_KEY = "s3native.replication";
- public static final short S3_NATIVE_REPLICATION_DEFAULT = 1;
- public static final String S3_NATIVE_STREAM_BUFFER_SIZE_KEY =
- "s3native.stream-buffer-size";
- public static final int S3_NATIVE_STREAM_BUFFER_SIZE_DEFAULT = 4096;
- public static final String S3_NATIVE_BYTES_PER_CHECKSUM_KEY =
- "s3native.bytes-per-checksum";
- public static final int S3_NATIVE_BYTES_PER_CHECKSUM_DEFAULT = 512;
- public static final String S3_NATIVE_CLIENT_WRITE_PACKET_SIZE_KEY =
- "s3native.client-write-packet-size";
- public static final int S3_NATIVE_CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024;
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/package.html
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/package.html b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/package.html
deleted file mode 100644
index 24b9b1d..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/package.html
+++ /dev/null
@@ -1,32 +0,0 @@
-<html>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<body>
-
-<p>
-A distributed implementation of {@link
-org.apache.hadoop.fs.FileSystem} for reading and writing files on
-<a href="http://aws.amazon.com/s3">Amazon S3</a>.
-Unlike {@link org.apache.hadoop.fs.s3.S3FileSystem}, which is block-based,
-this implementation stores
-files on S3 in their native form for interoperability with other S3 tools.
-</p>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
index 64632e4..17ffa7f 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
@@ -15,7 +15,5 @@
org.apache.hadoop.fs.LocalFileSystem
org.apache.hadoop.fs.viewfs.ViewFileSystem
-org.apache.hadoop.fs.s3.S3FileSystem
-org.apache.hadoop.fs.s3native.NativeS3FileSystem
org.apache.hadoop.fs.ftp.FTPFileSystem
org.apache.hadoop.fs.HarFileSystem
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
deleted file mode 100644
index ace6444..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
-
-/**
- * The contract of S3N: only enabled if the test bucket is provided
- */
-public class NativeS3Contract extends AbstractBondedFSContract {
-
- public static final String CONTRACT_XML = "contract/s3n.xml";
-
-
- public NativeS3Contract(Configuration conf) {
- super(conf);
- //insert the base features
- addConfResource(CONTRACT_XML);
- }
-
- @Override
- public String getScheme() {
- return "s3n";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java
deleted file mode 100644
index e44e2b1..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-
-public class TestS3NContractCreate extends AbstractContractCreateTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-
- @Override
- public void testOverwriteEmptyDirectory() throws Throwable {
- ContractTestUtils.skip(
- "blobstores can't distinguish empty directories from files");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java
deleted file mode 100644
index 1b79d27..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3NContractDelete extends AbstractContractDeleteTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java
deleted file mode 100644
index 527a31d..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * Test dir operations on S3
- */
-public class TestS3NContractMkdir extends AbstractContractMkdirTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java
deleted file mode 100644
index 2186f28..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3NContractOpen extends AbstractContractOpenTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java
deleted file mode 100644
index d673416..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3NContractRename extends AbstractContractRenameTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java
deleted file mode 100644
index 94f8483..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * root dir operations against an S3 bucket
- */
-public class TestS3NContractRootDir extends
- AbstractContractRootDirectoryTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java
deleted file mode 100644
index 6d04fff..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3NContractSeek extends AbstractContractSeekTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
deleted file mode 100644
index 2d43c8b..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.URI;
-import java.util.HashMap;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3.INode.FileType;
-
-/**
- * A stub implementation of {@link FileSystemStore} for testing
- * {@link S3FileSystem} without actually connecting to S3.
- */
-public class InMemoryFileSystemStore implements FileSystemStore {
-
- private Configuration conf;
- private SortedMap<Path, INode> inodes = new TreeMap<Path, INode>();
- private Map<Long, byte[]> blocks = new HashMap<Long, byte[]>();
-
- @Override
- public void initialize(URI uri, Configuration conf) {
- this.conf = conf;
- inodes.put(new Path("/"), INode.DIRECTORY_INODE);
- }
-
- @Override
- public String getVersion() throws IOException {
- return "0";
- }
-
- @Override
- public void deleteINode(Path path) throws IOException {
- inodes.remove(normalize(path));
- }
-
- @Override
- public void deleteBlock(Block block) throws IOException {
- blocks.remove(block.getId());
- }
-
- @Override
- public boolean inodeExists(Path path) throws IOException {
- return inodes.containsKey(normalize(path));
- }
-
- @Override
- public boolean blockExists(long blockId) throws IOException {
- return blocks.containsKey(blockId);
- }
-
- @Override
- public INode retrieveINode(Path path) throws IOException {
- return inodes.get(normalize(path));
- }
-
- @Override
- public File retrieveBlock(Block block, long byteRangeStart) throws IOException {
- byte[] data = blocks.get(block.getId());
- File file = createTempFile();
- BufferedOutputStream out = null;
- try {
- out = new BufferedOutputStream(new FileOutputStream(file));
- out.write(data, (int) byteRangeStart, data.length - (int) byteRangeStart);
- } finally {
- if (out != null) {
- out.close();
- }
- }
- return file;
- }
-
- private File createTempFile() throws IOException {
- File dir = new File(conf.get("fs.s3.buffer.dir"));
- if (!dir.exists() && !dir.mkdirs()) {
- throw new IOException("Cannot create S3 buffer directory: " + dir);
- }
- File result = File.createTempFile("test-", ".tmp", dir);
- result.deleteOnExit();
- return result;
- }
-
- @Override
- public Set<Path> listSubPaths(Path path) throws IOException {
- Path normalizedPath = normalize(path);
- // This is inefficient but more than adequate for testing purposes.
- Set<Path> subPaths = new LinkedHashSet<Path>();
- for (Path p : inodes.tailMap(normalizedPath).keySet()) {
- if (normalizedPath.equals(p.getParent())) {
- subPaths.add(p);
- }
- }
- return subPaths;
- }
-
- @Override
- public Set<Path> listDeepSubPaths(Path path) throws IOException {
- Path normalizedPath = normalize(path);
- String pathString = normalizedPath.toUri().getPath();
- if (!pathString.endsWith("/")) {
- pathString += "/";
- }
- // This is inefficient but more than adequate for testing purposes.
- Set<Path> subPaths = new LinkedHashSet<Path>();
- for (Path p : inodes.tailMap(normalizedPath).keySet()) {
- if (p.toUri().getPath().startsWith(pathString)) {
- subPaths.add(p);
- }
- }
- return subPaths;
- }
-
- @Override
- public void storeINode(Path path, INode inode) throws IOException {
- inodes.put(normalize(path), inode);
- }
-
- @Override
- public void storeBlock(Block block, File file) throws IOException {
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- byte[] buf = new byte[8192];
- int numRead;
- BufferedInputStream in = null;
- try {
- in = new BufferedInputStream(new FileInputStream(file));
- while ((numRead = in.read(buf)) >= 0) {
- out.write(buf, 0, numRead);
- }
- } finally {
- if (in != null) {
- in.close();
- }
- }
- blocks.put(block.getId(), out.toByteArray());
- }
-
- private Path normalize(Path path) {
- if (!path.isAbsolute()) {
- throw new IllegalArgumentException("Path must be absolute: " + path);
- }
- return new Path(path.toUri().getPath());
- }
-
- @Override
- public void purge() throws IOException {
- inodes.clear();
- blocks.clear();
- }
-
- @Override
- public void dump() throws IOException {
- StringBuilder sb = new StringBuilder(getClass().getSimpleName());
- sb.append(", \n");
- for (Map.Entry<Path, INode> entry : inodes.entrySet()) {
- sb.append(entry.getKey()).append("\n");
- INode inode = entry.getValue();
- sb.append("\t").append(inode.getFileType()).append("\n");
- if (inode.getFileType() == FileType.DIRECTORY) {
- continue;
- }
- for (int j = 0; j < inode.getBlocks().length; j++) {
- sb.append("\t").append(inode.getBlocks()[j]).append("\n");
- }
- }
- System.out.println(sb);
-
- System.out.println(inodes.keySet());
- System.out.println(blocks.keySet());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/Jets3tS3FileSystemContractTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/Jets3tS3FileSystemContractTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/Jets3tS3FileSystemContractTest.java
deleted file mode 100644
index 53b3c03..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/Jets3tS3FileSystemContractTest.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.IOException;
-
-public class Jets3tS3FileSystemContractTest
- extends S3FileSystemContractBaseTest {
-
- @Override
- FileSystemStore getFileSystemStore() throws IOException {
- return new Jets3tFileSystemStore();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java
deleted file mode 100644
index d704b00..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.IOException;
-import java.net.URI;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystemContractBaseTest;
-import org.apache.hadoop.fs.Path;
-
-public abstract class S3FileSystemContractBaseTest
- extends FileSystemContractBaseTest {
-
- private FileSystemStore store;
-
- abstract FileSystemStore getFileSystemStore() throws IOException;
-
- @Override
- protected void setUp() throws Exception {
- Configuration conf = new Configuration();
- store = getFileSystemStore();
- fs = new S3FileSystem(store);
- fs.initialize(URI.create(conf.get("test.fs.s3.name")), conf);
- }
-
- @Override
- protected void tearDown() throws Exception {
- store.purge();
- super.tearDown();
- }
-
- public void testBlockSize() throws Exception {
- Path file = path("/test/hadoop/file");
- long newBlockSize = fs.getDefaultBlockSize(file) * 2;
- fs.getConf().setLong("fs.s3.block.size", newBlockSize);
- createFile(file);
- assertEquals("Double default block size", newBlockSize,
- fs.getFileStatus(file).getBlockSize());
- }
-
- public void testCanonicalName() throws Exception {
- assertNull("s3 doesn't support security token and shouldn't have canonical name",
- fs.getCanonicalServiceName());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3InMemoryFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3InMemoryFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3InMemoryFileSystem.java
deleted file mode 100644
index a4e9770..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3InMemoryFileSystem.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import org.apache.hadoop.fs.s3.S3FileSystem;
-import org.apache.hadoop.fs.s3.InMemoryFileSystemStore;
-
-/**
- * A helper implementation of {@link S3FileSystem}
- * without actually connecting to S3 for unit testing.
- */
-public class S3InMemoryFileSystem extends S3FileSystem {
- public S3InMemoryFileSystem() {
- super(new InMemoryFileSystemStore());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestINode.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestINode.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestINode.java
deleted file mode 100644
index 086a43e..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/TestINode.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.fs.s3.INode.FileType;
-
-public class TestINode extends TestCase {
-
- public void testSerializeFileWithSingleBlock() throws IOException {
- Block[] blocks = { new Block(849282477840258181L, 128L) };
- INode inode = new INode(FileType.FILE, blocks);
-
- assertEquals("Length", 1L + 4 + 16, inode.getSerializedLength());
- InputStream in = inode.serialize();
-
- INode deserialized = INode.deserialize(in);
-
- assertEquals("FileType", inode.getFileType(), deserialized.getFileType());
- Block[] deserializedBlocks = deserialized.getBlocks();
- assertEquals("Length", 1, deserializedBlocks.length);
- assertEquals("Id", blocks[0].getId(), deserializedBlocks[0].getId());
- assertEquals("Length", blocks[0].getLength(), deserializedBlocks[0]
- .getLength());
-
- }
-
- public void testSerializeDirectory() throws IOException {
- INode inode = INode.DIRECTORY_INODE;
- assertEquals("Length", 1L, inode.getSerializedLength());
- InputStream in = inode.serialize();
- INode deserialized = INode.deserialize(in);
- assertSame(INode.DIRECTORY_INODE, deserialized);
- }
-
- public void testDeserializeNull() throws IOException {
- assertNull(INode.deserialize(null));
- }
-
-}
[04/20] git commit: YARN-1458. FairScheduler: Zero weight can lead to
livelock. (Zhihai Xu via kasha)
Posted by sz...@apache.org.
YARN-1458. FairScheduler: Zero weight can lead to livelock. (Zhihai Xu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3072c83b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3072c83b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3072c83b
Branch: refs/heads/HDFS-6584
Commit: 3072c83b38fd87318d502a7d1bc518963b5ccdf7
Parents: b100949
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Sep 10 08:26:14 2014 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Wed Sep 10 08:26:14 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../fair/policies/ComputeFairShares.java | 87 +++++++++--
.../scheduler/fair/TestFairScheduler.java | 145 ++++++++++++++++++-
3 files changed, 218 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3072c83b/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7eaf1c8..5734b98 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -305,6 +305,9 @@ Release 2.6.0 - UNRELEASED
YARN-2526. SLS can deadlock when all the threads are taken by AMSimulators.
(Wei Yan via kasha)
+ YARN-1458. FairScheduler: Zero weight can lead to livelock.
+ (Zhihai Xu via kasha)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3072c83b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
index 6836758..12ddde2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
@@ -48,16 +48,7 @@ public class ComputeFairShares {
public static void computeShares(
Collection<? extends Schedulable> schedulables, Resource totalResources,
ResourceType type) {
- Collection<Schedulable> activeSchedulables = new ArrayList<Schedulable>();
- for (Schedulable sched : schedulables) {
- if ((sched instanceof FSQueue) && !((FSQueue) sched).isActive()) {
- setResourceValue(0, sched.getFairShare(), type);
- } else {
- activeSchedulables.add(sched);
- }
- }
-
- computeSharesInternal(activeSchedulables, totalResources, type, false);
+ computeSharesInternal(schedulables, totalResources, type, false);
}
/**
@@ -117,8 +108,13 @@ public class ComputeFairShares {
* iterations of binary search is a constant (dependent on desired precision).
*/
private static void computeSharesInternal(
- Collection<? extends Schedulable> schedulables, Resource totalResources,
- ResourceType type, boolean isSteadyShare) {
+ Collection<? extends Schedulable> allSchedulables,
+ Resource totalResources, ResourceType type, boolean isSteadyShare) {
+
+ Collection<Schedulable> schedulables = new ArrayList<Schedulable>();
+ int takenResources = handleFixedFairShares(
+ allSchedulables, schedulables, isSteadyShare, type);
+
if (schedulables.isEmpty()) {
return;
}
@@ -135,9 +131,11 @@ public class ComputeFairShares {
totalMaxShare += maxShare;
}
}
- int totalResource = Math.min(totalMaxShare,
- getResourceValue(totalResources, type));
-
+
+ int totalResource = Math.max((getResourceValue(totalResources, type) -
+ takenResources), 0);
+ totalResource = Math.min(totalMaxShare, totalResource);
+
double rMax = 1.0;
while (resourceUsedWithWeightToResourceRatio(rMax, schedulables, type)
< totalResource) {
@@ -196,7 +194,64 @@ public class ComputeFairShares {
share = Math.min(share, getResourceValue(sched.getMaxShare(), type));
return (int) share;
}
-
+
+ /**
+ * Helper method to handle Schedulabes with fixed fairshares.
+ * Returns the resources taken by fixed fairshare schedulables,
+ * and adds the remaining to the passed nonFixedSchedulables.
+ */
+ private static int handleFixedFairShares(
+ Collection<? extends Schedulable> schedulables,
+ Collection<Schedulable> nonFixedSchedulables,
+ boolean isSteadyShare, ResourceType type) {
+ int totalResource = 0;
+
+ for (Schedulable sched : schedulables) {
+ int fixedShare = getFairShareIfFixed(sched, isSteadyShare, type);
+ if (fixedShare < 0) {
+ nonFixedSchedulables.add(sched);
+ } else {
+ setResourceValue(fixedShare,
+ isSteadyShare
+ ? ((FSQueue)sched).getSteadyFairShare()
+ : sched.getFairShare(),
+ type);
+ totalResource = (int) Math.min((long)totalResource + (long)fixedShare,
+ Integer.MAX_VALUE);
+ }
+ }
+ return totalResource;
+ }
+
+ /**
+ * Get the fairshare for the {@link Schedulable} if it is fixed, -1 otherwise.
+ *
+ * The fairshare is fixed if either the maxShare is 0, weight is 0,
+ * or the Schedulable is not active for instantaneous fairshare.
+ */
+ private static int getFairShareIfFixed(Schedulable sched,
+ boolean isSteadyShare, ResourceType type) {
+
+ // Check if maxShare is 0
+ if (getResourceValue(sched.getMaxShare(), type) <= 0) {
+ return 0;
+ }
+
+ // For instantaneous fairshares, check if queue is active
+ if (!isSteadyShare &&
+ (sched instanceof FSQueue) && !((FSQueue)sched).isActive()) {
+ return 0;
+ }
+
+ // Check if weight is 0
+ if (sched.getWeights().getWeight(type) <= 0) {
+ int minShare = getResourceValue(sched.getMinShare(), type);
+ return (minShare <= 0) ? 0 : minShare;
+ }
+
+ return -1;
+ }
+
private static int getResourceValue(Resource resource, ResourceType type) {
switch (type) {
case MEMORY:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3072c83b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 05b1925..3a0dac3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -307,7 +307,150 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals(3414, p.getMetrics().getSteadyFairShareMB());
}
}
-
+
+ @Test
+ public void testFairShareWithZeroWeight() throws IOException {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ // set queueA and queueB weight zero.
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queueA\">");
+ out.println("<weight>0.0</weight>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueB\">");
+ out.println("<weight>0.0</weight>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add one big node (only care about aggregate capacity)
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // Queue A wants 2 * 1024.
+ createSchedulingRequest(2 * 1024, "queueA", "user1");
+ // Queue B wants 6 * 1024
+ createSchedulingRequest(6 * 1024, "queueB", "user1");
+
+ scheduler.update();
+
+ FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue(
+ "queueA", false);
+ // queueA's weight is 0.0, so its fair share should be 0.
+ assertEquals(0, queue.getFairShare().getMemory());
+ // queueB's weight is 0.0, so its fair share should be 0.
+ queue = scheduler.getQueueManager().getLeafQueue(
+ "queueB", false);
+ assertEquals(0, queue.getFairShare().getMemory());
+ }
+
+ @Test
+ public void testFairShareWithZeroWeightNoneZeroMinRes() throws IOException {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ // set queueA and queueB weight zero.
+ // set queueA and queueB minResources 1.
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queueA\">");
+ out.println("<minResources>1 mb 1 vcores</minResources>");
+ out.println("<weight>0.0</weight>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueB\">");
+ out.println("<minResources>1 mb 1 vcores</minResources>");
+ out.println("<weight>0.0</weight>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add one big node (only care about aggregate capacity)
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // Queue A wants 2 * 1024.
+ createSchedulingRequest(2 * 1024, "queueA", "user1");
+ // Queue B wants 6 * 1024
+ createSchedulingRequest(6 * 1024, "queueB", "user1");
+
+ scheduler.update();
+
+ FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue(
+ "queueA", false);
+ // queueA's weight is 0.0 and minResources is 1,
+ // so its fair share should be 1 (minShare).
+ assertEquals(1, queue.getFairShare().getMemory());
+ // queueB's weight is 0.0 and minResources is 1,
+ // so its fair share should be 1 (minShare).
+ queue = scheduler.getQueueManager().getLeafQueue(
+ "queueB", false);
+ assertEquals(1, queue.getFairShare().getMemory());
+ }
+
+ @Test
+ public void testFairShareWithNoneZeroWeightNoneZeroMinRes()
+ throws IOException {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ // set queueA and queueB weight 0.5.
+ // set queueA and queueB minResources 1024.
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queueA\">");
+ out.println("<minResources>1024 mb 1 vcores</minResources>");
+ out.println("<weight>0.5</weight>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueB\">");
+ out.println("<minResources>1024 mb 1 vcores</minResources>");
+ out.println("<weight>0.5</weight>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add one big node (only care about aggregate capacity)
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // Queue A wants 4 * 1024.
+ createSchedulingRequest(4 * 1024, "queueA", "user1");
+ // Queue B wants 4 * 1024
+ createSchedulingRequest(4 * 1024, "queueB", "user1");
+
+ scheduler.update();
+
+ FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue(
+ "queueA", false);
+ // queueA's weight is 0.5 and minResources is 1024,
+ // so its fair share should be 4096.
+ assertEquals(4096, queue.getFairShare().getMemory());
+ // queueB's weight is 0.5 and minResources is 1024,
+ // so its fair share should be 4096.
+ queue = scheduler.getQueueManager().getLeafQueue(
+ "queueB", false);
+ assertEquals(4096, queue.getFairShare().getMemory());
+ }
+
@Test
public void testSimpleHierarchicalFairShareCalculation() throws IOException {
scheduler.init(conf);
[05/20] git commit: YARN-2448. Changed ApplicationMasterProtocol to
expose RM-recognized resource types to the AMs. Contributed by Varun Vasudev.
Posted by sz...@apache.org.
YARN-2448. Changed ApplicationMasterProtocol to expose RM-recognized resource types to the AMs. Contributed by Varun Vasudev.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b67d5ba7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b67d5ba7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b67d5ba7
Branch: refs/heads/HDFS-6584
Commit: b67d5ba7842cc10695d987f217027848a5a8c3d8
Parents: 3072c83
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Mon Sep 8 14:48:21 2014 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Wed Sep 10 10:15:47 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../RegisterApplicationMasterResponse.java | 23 ++++++
.../src/main/proto/yarn_service_protos.proto | 6 ++
...RegisterApplicationMasterResponsePBImpl.java | 78 ++++++++++++++++++--
.../ApplicationMasterService.java | 10 +--
.../scheduler/AbstractYarnScheduler.java | 15 ++--
.../scheduler/YarnScheduler.java | 10 +++
.../scheduler/capacity/CapacityScheduler.java | 22 ++++--
.../scheduler/fair/FairScheduler.java | 16 ++--
.../TestApplicationMasterService.java | 53 ++++++++++++-
10 files changed, 202 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67d5ba7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5734b98..a840e4f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -196,6 +196,9 @@ Release 2.6.0 - UNRELEASED
YARN-2515. Updated ConverterUtils#toContainerId to parse epoch.
(Tsuyoshi OZAWA via jianhe)
+ YARN-2448. Changed ApplicationMasterProtocol to expose RM-recognized resource
+ types to the AMs. (Varun Vasudev via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67d5ba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
index 79f9f3a..33daf28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.api.protocolrecords;
import java.nio.ByteBuffer;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -31,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.util.Records;
/**
@@ -180,4 +182,25 @@ public abstract class RegisterApplicationMasterResponse {
@Private
@Unstable
public abstract void setNMTokensFromPreviousAttempts(List<NMToken> nmTokens);
+
+ /**
+ * Get a set of the resource types considered by the scheduler.
+ *
+ * @return a Map of RM settings
+ */
+ @Public
+ @Unstable
+ public abstract EnumSet<SchedulerResourceTypes> getSchedulerResourceTypes();
+
+ /**
+ * Set the resource types used by the scheduler.
+ *
+ * @param types
+ * a set of the resource types that the scheduler considers during
+ * scheduling
+ */
+ @Private
+ @Unstable
+ public abstract void setSchedulerResourceTypes(
+ EnumSet<SchedulerResourceTypes> types);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67d5ba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index df8784b..4203744 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -47,6 +47,7 @@ message RegisterApplicationMasterResponseProto {
repeated ContainerProto containers_from_previous_attempts = 4;
optional string queue = 5;
repeated NMTokenProto nm_tokens_from_previous_attempts = 6;
+ repeated SchedulerResourceTypes scheduler_resource_types = 7;
}
message FinishApplicationMasterRequestProto {
@@ -88,6 +89,11 @@ message AllocateResponseProto {
optional hadoop.common.TokenProto am_rm_token = 12;
}
+enum SchedulerResourceTypes {
+ MEMORY = 0;
+ CPU = 1;
+}
+
//////////////////////////////////////////////////////
/////// client_RM_Protocol ///////////////////////////
//////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67d5ba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
index 06a637a..32dc85d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
@@ -20,11 +20,7 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -43,6 +39,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.NMTokenProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import com.google.protobuf.ByteString;
import com.google.protobuf.TextFormat;
@@ -61,6 +58,7 @@ public class RegisterApplicationMasterResponsePBImpl extends
private Map<ApplicationAccessType, String> applicationACLS = null;
private List<Container> containersFromPreviousAttempts = null;
private List<NMToken> nmTokens = null;
+ private EnumSet<SchedulerResourceTypes> schedulerResourceTypes = null;
public RegisterApplicationMasterResponsePBImpl() {
builder = RegisterApplicationMasterResponseProto.newBuilder();
@@ -122,6 +120,9 @@ public class RegisterApplicationMasterResponsePBImpl extends
Iterable<NMTokenProto> iterable = getTokenProtoIterable(nmTokens);
builder.addAllNmTokensFromPreviousAttempts(iterable);
}
+ if(schedulerResourceTypes != null) {
+ addSchedulerResourceTypes();
+ }
}
@@ -364,6 +365,73 @@ public class RegisterApplicationMasterResponsePBImpl extends
};
}
+ @Override
+ public EnumSet<SchedulerResourceTypes> getSchedulerResourceTypes() {
+ initSchedulerResourceTypes();
+ return this.schedulerResourceTypes;
+ }
+
+ private void initSchedulerResourceTypes() {
+ if (this.schedulerResourceTypes != null) {
+ return;
+ }
+ RegisterApplicationMasterResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+
+ List<SchedulerResourceTypes> list = p.getSchedulerResourceTypesList();
+ if (list.isEmpty()) {
+ this.schedulerResourceTypes =
+ EnumSet.noneOf(SchedulerResourceTypes.class);
+ } else {
+ this.schedulerResourceTypes = EnumSet.copyOf(list);
+ }
+ }
+
+ private void addSchedulerResourceTypes() {
+ maybeInitBuilder();
+ builder.clearSchedulerResourceTypes();
+ if (schedulerResourceTypes == null) {
+ return;
+ }
+ Iterable<? extends SchedulerResourceTypes> values =
+ new Iterable<SchedulerResourceTypes>() {
+
+ @Override
+ public Iterator<SchedulerResourceTypes> iterator() {
+ return new Iterator<SchedulerResourceTypes>() {
+ Iterator<SchedulerResourceTypes> settingsIterator =
+ schedulerResourceTypes.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return settingsIterator.hasNext();
+ }
+
+ @Override
+ public SchedulerResourceTypes next() {
+ return settingsIterator.next();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ };
+ this.builder.addAllSchedulerResourceTypes(values);
+ }
+
+ @Override
+ public void setSchedulerResourceTypes(EnumSet<SchedulerResourceTypes> types) {
+ if (types == null) {
+ return;
+ }
+ initSchedulerResourceTypes();
+ this.schedulerResourceTypes.clear();
+ this.schedulerResourceTypes.addAll(types);
+ }
+
private Resource convertFromProtoFormat(ResourceProto resource) {
return new ResourcePBImpl(resource);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67d5ba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index d77180c..e6d878a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -22,11 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -329,6 +325,10 @@ public class ApplicationMasterService extends AbstractService implements
+ transferredContainers.size() + " containers from previous"
+ " attempts and " + nmTokens.size() + " NM tokens.");
}
+
+ response.setSchedulerResourceTypes(rScheduler
+ .getSchedulingResourceTypes());
+
return response;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67d5ba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index ee5dcbe..0b5447b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -19,13 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.logging.Log;
@@ -45,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
@@ -502,4 +497,10 @@ public abstract class AbstractYarnScheduler
+ " with the same resource: " + newResource);
}
}
+
+ /** {@inheritDoc} */
+ @Override
+ public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes() {
+ return EnumSet.of(SchedulerResourceTypes.MEMORY);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67d5ba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 5ce16c2..b6c1018 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.io.IOException;
+import java.util.EnumSet;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
@@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
/**
* This interface is used by the components to talk to the
@@ -220,4 +222,12 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
* @throws YarnException
*/
void killAllAppsInQueue(String queueName) throws YarnException;
+
+ /**
+ * Return a collection of the resource types that are considered when
+ * scheduling
+ *
+ * @return an EnumSet containing the resource types
+ */
+ public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67d5ba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index a8ef942..6b810d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -20,13 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -50,12 +44,12 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.*;
@@ -89,6 +83,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.Lock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -1285,4 +1280,15 @@ public class CapacityScheduler extends
}
return (LeafQueue) ret;
}
+
+ /** {@inheritDoc} */
+ @Override
+ public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes() {
+ if (calculator.getClass().getName()
+ .equals(DefaultResourceCalculator.class.getName())) {
+ return EnumSet.of(SchedulerResourceTypes.MEMORY);
+ }
+ return EnumSet
+ .of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67d5ba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index a35e49f..9c40d48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -19,13 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.logging.Log;
@@ -50,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
@@ -1529,4 +1524,11 @@ public class FairScheduler extends
queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
queueMgr.getRootQueue().recomputeSteadyShares();
}
+
+ /** {@inheritDoc} */
+ @Override
+ public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes() {
+ return EnumSet
+ .of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67d5ba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index b0ffc85..3508a3c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -18,6 +18,13 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -40,8 +47,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
import static java.lang.Thread.sleep;
import static org.mockito.Matchers.any;
@@ -259,4 +265,47 @@ public class TestApplicationMasterService {
}
}
}
+
+ @Test(timeout = 3000000)
+ public void testResourceTypes() throws Exception {
+ HashMap<YarnConfiguration, EnumSet<SchedulerResourceTypes>> driver =
+ new HashMap<YarnConfiguration, EnumSet<SchedulerResourceTypes>>();
+
+ CapacitySchedulerConfiguration csconf =
+ new CapacitySchedulerConfiguration();
+ csconf.setResourceComparator(DominantResourceCalculator.class);
+ YarnConfiguration testCapacityDRConf = new YarnConfiguration(csconf);
+ testCapacityDRConf.setClass(YarnConfiguration.RM_SCHEDULER,
+ CapacityScheduler.class, ResourceScheduler.class);
+ YarnConfiguration testCapacityDefConf = new YarnConfiguration();
+ testCapacityDefConf.setClass(YarnConfiguration.RM_SCHEDULER,
+ CapacityScheduler.class, ResourceScheduler.class);
+ YarnConfiguration testFairDefConf = new YarnConfiguration();
+ testFairDefConf.setClass(YarnConfiguration.RM_SCHEDULER,
+ FairScheduler.class, ResourceScheduler.class);
+
+ driver.put(conf, EnumSet.of(SchedulerResourceTypes.MEMORY));
+ driver.put(testCapacityDRConf,
+ EnumSet.of(SchedulerResourceTypes.CPU, SchedulerResourceTypes.MEMORY));
+ driver.put(testCapacityDefConf, EnumSet.of(SchedulerResourceTypes.MEMORY));
+ driver.put(testFairDefConf,
+ EnumSet.of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU));
+
+ for (Map.Entry<YarnConfiguration, EnumSet<SchedulerResourceTypes>> entry : driver
+ .entrySet()) {
+ EnumSet<SchedulerResourceTypes> expectedValue = entry.getValue();
+ MockRM rm = new MockRM(entry.getKey());
+ rm.start();
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
+ RMApp app1 = rm.submitApp(2048);
+ nm1.nodeHeartbeat(true);
+ RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+ MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+ RegisterApplicationMasterResponse resp = am1.registerAppAttempt();
+ EnumSet<SchedulerResourceTypes> types = resp.getSchedulerResourceTypes();
+ LOG.info("types = " + types.toString());
+ Assert.assertEquals(expectedValue, types);
+ rm.stop();
+ }
+ }
}
[20/20] git commit: Merge branch 'trunk' into HDFS-6584
Posted by sz...@apache.org.
Merge branch 'trunk' into HDFS-6584
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/70dfe9cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/70dfe9cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/70dfe9cf
Branch: refs/heads/HDFS-6584
Commit: 70dfe9cfab73fc1ff0ed4dd37027a77b19ac3104
Parents: db41a1b 4be9517
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Thu Sep 11 15:23:38 2014 +0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Thu Sep 11 15:23:38 2014 +0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 7 +
.../java/org/apache/hadoop/fs/s3/Block.java | 52 --
.../apache/hadoop/fs/s3/FileSystemStore.java | 67 --
.../java/org/apache/hadoop/fs/s3/INode.java | 128 ---
.../hadoop/fs/s3/Jets3tFileSystemStore.java | 445 -----------
.../org/apache/hadoop/fs/s3/MigrationTool.java | 291 -------
.../org/apache/hadoop/fs/s3/S3Credentials.java | 103 ---
.../org/apache/hadoop/fs/s3/S3Exception.java | 39 -
.../org/apache/hadoop/fs/s3/S3FileSystem.java | 486 -----------
.../hadoop/fs/s3/S3FileSystemConfigKeys.java | 47 --
.../hadoop/fs/s3/S3FileSystemException.java | 36 -
.../org/apache/hadoop/fs/s3/S3InputStream.java | 215 -----
.../org/apache/hadoop/fs/s3/S3OutputStream.java | 235 ------
.../hadoop/fs/s3/VersionMismatchException.java | 37 -
.../java/org/apache/hadoop/fs/s3/package.html | 55 --
.../apache/hadoop/fs/s3native/FileMetadata.java | 59 --
.../s3native/Jets3tNativeFileSystemStore.java | 483 -----------
.../fs/s3native/NativeFileSystemStore.java | 67 --
.../hadoop/fs/s3native/NativeS3FileSystem.java | 796 -------------------
.../hadoop/fs/s3native/PartialListing.java | 64 --
.../s3native/S3NativeFileSystemConfigKeys.java | 47 --
.../org/apache/hadoop/fs/s3native/package.html | 32 -
.../authorize/DefaultImpersonationProvider.java | 2 +-
.../hadoop/util/GenericOptionsParser.java | 2 +-
.../services/org.apache.hadoop.fs.FileSystem | 2 -
.../fs/contract/s3n/NativeS3Contract.java | 43 -
.../fs/contract/s3n/TestS3NContractCreate.java | 38 -
.../fs/contract/s3n/TestS3NContractDelete.java | 31 -
.../fs/contract/s3n/TestS3NContractMkdir.java | 34 -
.../fs/contract/s3n/TestS3NContractOpen.java | 31 -
.../fs/contract/s3n/TestS3NContractRename.java | 32 -
.../fs/contract/s3n/TestS3NContractRootDir.java | 35 -
.../fs/contract/s3n/TestS3NContractSeek.java | 31 -
.../hadoop/fs/s3/InMemoryFileSystemStore.java | 200 -----
.../fs/s3/Jets3tS3FileSystemContractTest.java | 31 -
.../fs/s3/S3FileSystemContractBaseTest.java | 63 --
.../hadoop/fs/s3/S3InMemoryFileSystem.java | 32 -
.../java/org/apache/hadoop/fs/s3/TestINode.java | 60 --
.../fs/s3/TestInMemoryS3FileSystemContract.java | 31 -
.../apache/hadoop/fs/s3/TestS3Credentials.java | 36 -
.../apache/hadoop/fs/s3/TestS3FileSystem.java | 50 --
.../hadoop/fs/s3/TestS3InMemoryFileSystem.java | 67 --
.../s3native/InMemoryNativeFileSystemStore.java | 206 -----
.../Jets3tNativeS3FileSystemContractTest.java | 30 -
.../NativeS3FileSystemContractBaseTest.java | 233 ------
.../fs/s3native/S3NInMemoryFileSystem.java | 32 -
.../TestInMemoryNativeS3FileSystemContract.java | 30 -
.../TestJets3tNativeFileSystemStore.java | 126 ---
.../fs/s3native/TestS3NInMemoryFileSystem.java | 69 --
.../security/authorize/TestProxyUsers.java | 15 +
.../hadoop/util/TestGenericOptionsParser.java | 2 +-
.../src/test/resources/contract/s3n.xml | 95 ---
.../hadoop-kms/src/main/conf/kms-acls.xml | 38 +
.../hadoop/crypto/key/kms/server/KMSACLs.java | 97 ++-
.../crypto/key/kms/server/KMSConfiguration.java | 9 +
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 17 +-
.../kms/server/KeyAuthorizationKeyProvider.java | 276 +++++++
.../hadoop-kms/src/site/apt/index.apt.vm | 106 +++
.../hadoop/crypto/key/kms/server/TestKMS.java | 236 +++++-
.../server/TestKeyAuthorizationKeyProvider.java | 218 +++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../DelegationTokenSecretManager.java | 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 3 +
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 18 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 41 +
hadoop-mapreduce-project/CHANGES.txt | 3 +
...istoryServerFileSystemStateStoreService.java | 6 +
hadoop-project/pom.xml | 12 +
.../hadoop-aws/dev-support/findbugs-exclude.xml | 358 ++++++++-
.../java/org/apache/hadoop/fs/s3/Block.java | 52 ++
.../apache/hadoop/fs/s3/FileSystemStore.java | 67 ++
.../java/org/apache/hadoop/fs/s3/INode.java | 128 +++
.../hadoop/fs/s3/Jets3tFileSystemStore.java | 445 +++++++++++
.../org/apache/hadoop/fs/s3/MigrationTool.java | 291 +++++++
.../org/apache/hadoop/fs/s3/S3Credentials.java | 103 +++
.../org/apache/hadoop/fs/s3/S3Exception.java | 39 +
.../org/apache/hadoop/fs/s3/S3FileSystem.java | 486 +++++++++++
.../hadoop/fs/s3/S3FileSystemConfigKeys.java | 47 ++
.../hadoop/fs/s3/S3FileSystemException.java | 36 +
.../org/apache/hadoop/fs/s3/S3InputStream.java | 215 +++++
.../org/apache/hadoop/fs/s3/S3OutputStream.java | 235 ++++++
.../hadoop/fs/s3/VersionMismatchException.java | 37 +
.../java/org/apache/hadoop/fs/s3/package.html | 55 ++
.../apache/hadoop/fs/s3native/FileMetadata.java | 59 ++
.../s3native/Jets3tNativeFileSystemStore.java | 483 +++++++++++
.../fs/s3native/NativeFileSystemStore.java | 67 ++
.../hadoop/fs/s3native/NativeS3FileSystem.java | 796 +++++++++++++++++++
.../hadoop/fs/s3native/PartialListing.java | 64 ++
.../s3native/S3NativeFileSystemConfigKeys.java | 47 ++
.../org/apache/hadoop/fs/s3native/package.html | 32 +
.../services/org.apache.hadoop.fs.FileSystem | 17 +
.../fs/contract/s3n/NativeS3Contract.java | 43 +
.../fs/contract/s3n/TestS3NContractCreate.java | 38 +
.../fs/contract/s3n/TestS3NContractDelete.java | 31 +
.../fs/contract/s3n/TestS3NContractMkdir.java | 34 +
.../fs/contract/s3n/TestS3NContractOpen.java | 31 +
.../fs/contract/s3n/TestS3NContractRename.java | 32 +
.../fs/contract/s3n/TestS3NContractRootDir.java | 35 +
.../fs/contract/s3n/TestS3NContractSeek.java | 31 +
.../hadoop/fs/s3/InMemoryFileSystemStore.java | 200 +++++
.../fs/s3/Jets3tS3FileSystemContractTest.java | 31 +
.../fs/s3/S3FileSystemContractBaseTest.java | 54 ++
.../hadoop/fs/s3/S3InMemoryFileSystem.java | 32 +
.../java/org/apache/hadoop/fs/s3/TestINode.java | 60 ++
.../fs/s3/TestInMemoryS3FileSystemContract.java | 31 +
.../apache/hadoop/fs/s3/TestS3Credentials.java | 36 +
.../apache/hadoop/fs/s3/TestS3FileSystem.java | 50 ++
.../hadoop/fs/s3/TestS3InMemoryFileSystem.java | 67 ++
.../s3native/InMemoryNativeFileSystemStore.java | 206 +++++
.../Jets3tNativeS3FileSystemContractTest.java | 30 +
.../NativeS3FileSystemContractBaseTest.java | 233 ++++++
.../fs/s3native/S3NInMemoryFileSystem.java | 32 +
.../TestInMemoryNativeS3FileSystemContract.java | 30 +
.../TestJets3tNativeFileSystemStore.java | 126 +++
.../fs/s3native/TestS3NInMemoryFileSystem.java | 69 ++
.../src/test/resources/contract/s3n.xml | 95 +++
hadoop-tools/hadoop-tools-dist/pom.xml | 6 +
hadoop-yarn-project/CHANGES.txt | 18 +
.../RegisterApplicationMasterResponse.java | 23 +
.../records/ApplicationResourceUsageReport.java | 41 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 6 +
.../src/main/proto/yarn_protos.proto | 2 +
.../src/main/proto/yarn_service_protos.proto | 6 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 6 +
.../hadoop/yarn/client/cli/TestYarnCLI.java | 8 +-
...RegisterApplicationMasterResponsePBImpl.java | 78 +-
.../ApplicationResourceUsageReportPBImpl.java | 24 +
.../src/main/resources/yarn-default.xml | 16 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 5 +-
.../util/CgroupsLCEResourcesHandler.java | 94 ++-
.../util/NodeManagerHardwareUtils.java | 79 ++
.../util/TestCgroupsLCEResourcesHandler.java | 147 +++-
.../util/TestNodeManagerHardwareUtils.java | 72 ++
.../ApplicationMasterService.java | 10 +-
.../server/resourcemanager/RMAppManager.java | 2 +-
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../recovery/FileSystemRMStateStore.java | 4 +-
.../recovery/MemoryRMStateStore.java | 9 +-
.../resourcemanager/recovery/RMStateStore.java | 24 +-
.../recovery/ZKRMStateStore.java | 4 +-
.../records/ApplicationAttemptStateData.java | 32 +-
.../pb/ApplicationAttemptStateDataPBImpl.java | 24 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 23 +-
.../resourcemanager/rmapp/RMAppMetrics.java | 15 +-
.../attempt/AggregateAppResourceUsage.java | 60 ++
.../rmapp/attempt/RMAppAttemptImpl.java | 16 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 36 +-
.../rmcontainer/RMContainerImpl.java | 26 +-
.../scheduler/AbstractYarnScheduler.java | 15 +-
.../scheduler/SchedulerApplicationAttempt.java | 41 +-
.../scheduler/YarnScheduler.java | 10 +
.../scheduler/capacity/CapacityScheduler.java | 22 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 3 +
.../scheduler/fair/FSAppAttempt.java | 3 +
.../scheduler/fair/FairScheduler.java | 16 +-
.../fair/policies/ComputeFairShares.java | 87 +-
.../server/resourcemanager/webapp/AppBlock.java | 5 +-
.../resourcemanager/webapp/dao/AppInfo.java | 12 +
.../yarn_server_resourcemanager_recovery.proto | 4 +-
.../TestApplicationMasterService.java | 53 +-
.../resourcemanager/TestClientRMService.java | 60 +-
.../TestContainerResourceUsage.java | 401 ++++++++++
.../server/resourcemanager/TestRMRestart.java | 50 ++
.../applicationsmanager/MockAsm.java | 19 +-
.../recovery/RMStateStoreTestBase.java | 18 +-
.../rmapp/TestRMAppTransitions.java | 36 +-
.../attempt/TestRMAppAttemptTransitions.java | 72 +-
.../rmcontainer/TestRMContainerImpl.java | 11 +
.../scheduler/capacity/TestLeafQueue.java | 89 ++-
.../scheduler/fair/FairSchedulerTestBase.java | 4 +-
.../scheduler/fair/TestFairScheduler.java | 145 +++-
.../webapp/TestRMWebAppFairScheduler.java | 2 +-
.../webapp/TestRMWebServicesApps.java | 2 +-
.../TestRMWebServicesAppsModification.java | 50 +-
.../src/site/apt/ResourceManagerRest.apt.vm | 22 +-
175 files changed, 8886 insertions(+), 5666 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70dfe9cf/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/70dfe9cf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
[06/20] git commit: YARN-2459. RM crashes if App gets rejected for
any reason and HA is enabled. Contributed by Jian He
Posted by sz...@apache.org.
YARN-2459. RM crashes if App gets rejected for any reason and HA is enabled. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47bdfa04
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47bdfa04
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47bdfa04
Branch: refs/heads/HDFS-6584
Commit: 47bdfa044aa1d587b24edae8b1b0c796d829c960
Parents: b67d5ba
Author: XUAN <xg...@apache.org>
Authored: Wed Sep 10 11:44:41 2014 -0700
Committer: XUAN <xg...@apache.org>
Committed: Wed Sep 10 11:44:41 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../server/resourcemanager/RMAppManager.java | 2 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 6 ++-
.../server/resourcemanager/TestRMRestart.java | 50 ++++++++++++++++++++
.../rmapp/TestRMAppTransitions.java | 20 +++++++-
5 files changed, 77 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47bdfa04/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a840e4f..2feae46 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -311,6 +311,9 @@ Release 2.6.0 - UNRELEASED
YARN-1458. FairScheduler: Zero weight can lead to livelock.
(Zhihai Xu via kasha)
+ YARN-2459. RM crashes if App gets rejected for any reason
+ and HA is enabled. (Jian He via xgong)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47bdfa04/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 51024cf..a789e92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -401,7 +401,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
}
}
- private Credentials parseCredentials(ApplicationSubmissionContext application)
+ protected Credentials parseCredentials(ApplicationSubmissionContext application)
throws IOException {
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47bdfa04/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 48cf460..0b81f96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -150,8 +150,10 @@ public class RMAppImpl implements RMApp, Recoverable {
RMAppEventType.RECOVER, new RMAppRecoveredTransition())
.addTransition(RMAppState.NEW, RMAppState.KILLED, RMAppEventType.KILL,
new AppKilledTransition())
- .addTransition(RMAppState.NEW, RMAppState.FAILED,
- RMAppEventType.APP_REJECTED, new AppRejectedTransition())
+ .addTransition(RMAppState.NEW, RMAppState.FINAL_SAVING,
+ RMAppEventType.APP_REJECTED,
+ new FinalSavingTransition(new AppRejectedTransition(),
+ RMAppState.FAILED))
// Transitions from NEW_SAVING state
.addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47bdfa04/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 5f63caf..7d511db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -92,6 +93,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.log4j.Level;
@@ -1606,6 +1609,53 @@ public class TestRMRestart {
Assert.assertEquals(2, ((TestMemoryRMStateStore) memStore).updateApp);
}
+ // Test Application that fails on submission is saved in state store.
+ @Test (timeout = 20000)
+ public void testAppFailedOnSubmissionSavedInStateStore() throws Exception {
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+
+ MockRM rm1 = new TestSecurityMockRM(conf, memStore) {
+ @Override
+ protected RMAppManager createRMAppManager() {
+ return new TestRMAppManager(this.rmContext, this.scheduler,
+ this.masterService, this.applicationACLsManager, conf);
+ }
+
+ class TestRMAppManager extends RMAppManager {
+
+ public TestRMAppManager(RMContext context, YarnScheduler scheduler,
+ ApplicationMasterService masterService,
+ ApplicationACLsManager applicationACLsManager, Configuration conf) {
+ super(context, scheduler, masterService, applicationACLsManager, conf);
+ }
+
+ @Override
+ protected Credentials parseCredentials(
+ ApplicationSubmissionContext application) throws IOException {
+ throw new IOException("Parsing credential error.");
+ }
+ }
+ };
+ rm1.start();
+ RMApp app1 =
+ rm1.submitApp(200, "name", "user",
+ new HashMap<ApplicationAccessType, String>(), false, "default", -1,
+ null, "MAPREDUCE", false);
+ rm1.waitForState(app1.getApplicationId(), RMAppState.FAILED);
+ // Check app staet is saved in state store.
+ Assert.assertEquals(RMAppState.FAILED, memStore.getState()
+ .getApplicationState().get(app1.getApplicationId()).getState());
+
+ MockRM rm2 = new TestSecurityMockRM(conf, memStore);
+ rm2.start();
+ // Restarted RM has the failed app info too.
+ rm2.waitForState(app1.getApplicationId(), RMAppState.FAILED);
+ }
+
@SuppressWarnings("resource")
@Test (timeout = 60000)
public void testQueueMetricsOnRMRestart() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47bdfa04/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 2fc4431..3c871df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -526,8 +526,26 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText);
- assertAppFinalStateNotSaved(application);
+ assertAppFinalStateSaved(application);
+ verifyApplicationFinished(RMAppState.FAILED);
+ }
+
+ @Test (timeout = 30000)
+ public void testAppNewRejectAddToStore() throws IOException {
+ LOG.info("--- START: testAppNewRejectAddToStore ---");
+
+ RMApp application = createNewTestApp(null);
+ // NEW => FAILED event RMAppEventType.APP_REJECTED
+ String rejectedText = "Test Application Rejected";
+ RMAppEvent event =
+ new RMAppRejectedEvent(application.getApplicationId(), rejectedText);
+ application.handle(event);
+ rmDispatcher.await();
+ sendAppUpdateSavedEvent(application);
+ assertFailed(application, rejectedText);
+ assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.FAILED);
+ rmContext.getStateStore().removeApplication(application);
}
@Test (timeout = 30000)
[03/20] git commit: HADOOP-9989. Bug introduced in HADOOP-9374,
which parses the -tokenCacheFile as binary file but set it to the
configuration as JSON file. (zxu via tucu)
Posted by sz...@apache.org.
HADOOP-9989. Bug introduced in HADOOP-9374, which parses the -tokenCacheFile as binary file but set it to the configuration as JSON file. (zxu via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1009494
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1009494
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1009494
Branch: refs/heads/HDFS-6584
Commit: b100949404843ed245ef4e118291f55b3fdc81b8
Parents: 9ee891a
Author: Alejandro Abdelnur <tu...@apache.org>
Authored: Tue Sep 9 22:19:42 2014 -0700
Committer: Alejandro Abdelnur <tu...@apache.org>
Committed: Tue Sep 9 22:19:42 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/util/GenericOptionsParser.java | 2 +-
.../java/org/apache/hadoop/util/TestGenericOptionsParser.java | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1009494/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index b015087..b2157d6 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -779,6 +779,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11077. NPE if hosts not specified in ProxyUsers. (gchanan via tucu)
+ HADOOP-9989. Bug introduced in HADOOP-9374, which parses the -tokenCacheFile
+ as binary file but set it to the configuration as JSON file. (zxu via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1009494/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
index 18acbf1..2a37dac 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
@@ -332,7 +332,7 @@ public class GenericOptionsParser {
}
UserGroupInformation.getCurrentUser().addCredentials(
Credentials.readTokenStorageFile(p, conf));
- conf.set("mapreduce.job.credentials.json", p.toString(),
+ conf.set("mapreduce.job.credentials.binary", p.toString(),
"from -tokenCacheFile command line option");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1009494/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
index 779318a..2bc1915 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
@@ -249,7 +249,7 @@ public class TestGenericOptionsParser extends TestCase {
creds.writeTokenStorageFile(tmpPath, conf);
new GenericOptionsParser(conf, args);
- String fileName = conf.get("mapreduce.job.credentials.json");
+ String fileName = conf.get("mapreduce.job.credentials.binary");
assertNotNull("files is null", fileName);
assertEquals("files option does not match", tmpPath.toString(), fileName);
[09/20] git commit: Fix CHANGES.txt. Credit Mayank Bansal for his
contributions on YARN-2459
Posted by sz...@apache.org.
Fix CHANGES.txt. Credit Mayank Bansal for his contributions on YARN-2459
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7d38ffc8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7d38ffc8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7d38ffc8
Branch: refs/heads/HDFS-6584
Commit: 7d38ffc8d3500d428bdad5640e9e70d66ed5ea60
Parents: b02a4b4
Author: XuanGong <xg...@apache.org>
Authored: Wed Sep 10 15:14:47 2014 -0700
Committer: XuanGong <xg...@apache.org>
Committed: Wed Sep 10 15:14:47 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d38ffc8/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cf5ec65..7968071 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -312,7 +312,7 @@ Release 2.6.0 - UNRELEASED
(Zhihai Xu via kasha)
YARN-2459. RM crashes if App gets rejected for any reason
- and HA is enabled. (Jian He via xgong)
+ and HA is enabled. (Jian He and Mayank Bansal via xgong)
YARN-2158. Fixed TestRMWebServicesAppsModification#testSingleAppKill test
failure. (Varun Vasudev via jianhe)
[11/20] HADOOP-11074. Move s3-related FS connector code to
hadoop-aws. (David S. Wang via Colin Patrick McCabe)
Posted by sz...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
new file mode 100644
index 0000000..f6f9ae9
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3native.NativeS3FileSystem.NativeS3FsInputStream;
+
+public abstract class NativeS3FileSystemContractBaseTest
+ extends FileSystemContractBaseTest {
+
+ private NativeFileSystemStore store;
+
+ abstract NativeFileSystemStore getNativeFileSystemStore() throws IOException;
+
+ @Override
+ protected void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ store = getNativeFileSystemStore();
+ fs = new NativeS3FileSystem(store);
+ fs.initialize(URI.create(conf.get("test.fs.s3n.name")), conf);
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ store.purge("test");
+ super.tearDown();
+ }
+
+ public void testCanonicalName() throws Exception {
+ assertNull("s3n doesn't support security token and shouldn't have canonical name",
+ fs.getCanonicalServiceName());
+ }
+
+ public void testListStatusForRoot() throws Exception {
+ FileStatus[] paths = fs.listStatus(path("/"));
+ assertEquals("Root directory is not empty; ", 0, paths.length);
+
+ Path testDir = path("/test");
+ assertTrue(fs.mkdirs(testDir));
+
+ paths = fs.listStatus(path("/"));
+ assertEquals(1, paths.length);
+ assertEquals(path("/test"), paths[0].getPath());
+ }
+
+ public void testNoTrailingBackslashOnBucket() throws Exception {
+ assertTrue(fs.getFileStatus(new Path(fs.getUri().toString())).isDirectory());
+ }
+
+ private void createTestFiles(String base) throws IOException {
+ store.storeEmptyFile(base + "/file1");
+ store.storeEmptyFile(base + "/dir/file2");
+ store.storeEmptyFile(base + "/dir/file3");
+ }
+
+ public void testDirWithDifferentMarkersWorks() throws Exception {
+
+ for (int i = 0; i < 3; i++) {
+ String base = "test/hadoop" + i;
+ Path path = path("/" + base);
+
+ createTestFiles(base);
+
+ if (i == 0 ) {
+ //do nothing, we are testing correctness with no markers
+ }
+ else if (i == 1) {
+ // test for _$folder$ marker
+ store.storeEmptyFile(base + "_$folder$");
+ store.storeEmptyFile(base + "/dir_$folder$");
+ }
+ else if (i == 2) {
+ // test the end slash file marker
+ store.storeEmptyFile(base + "/");
+ store.storeEmptyFile(base + "/dir/");
+ }
+ else if (i == 3) {
+ // test both markers
+ store.storeEmptyFile(base + "_$folder$");
+ store.storeEmptyFile(base + "/dir_$folder$");
+ store.storeEmptyFile(base + "/");
+ store.storeEmptyFile(base + "/dir/");
+ }
+
+ assertTrue(fs.getFileStatus(path).isDirectory());
+ assertEquals(2, fs.listStatus(path).length);
+ }
+ }
+
+ public void testDeleteWithNoMarker() throws Exception {
+ String base = "test/hadoop";
+ Path path = path("/" + base);
+
+ createTestFiles(base);
+
+ fs.delete(path, true);
+
+ path = path("/test");
+ assertTrue(fs.getFileStatus(path).isDirectory());
+ assertEquals(0, fs.listStatus(path).length);
+ }
+
+ public void testRenameWithNoMarker() throws Exception {
+ String base = "test/hadoop";
+ Path dest = path("/test/hadoop2");
+
+ createTestFiles(base);
+
+ fs.rename(path("/" + base), dest);
+
+ Path path = path("/test");
+ assertTrue(fs.getFileStatus(path).isDirectory());
+ assertEquals(1, fs.listStatus(path).length);
+ assertTrue(fs.getFileStatus(dest).isDirectory());
+ assertEquals(2, fs.listStatus(dest).length);
+ }
+
+ public void testEmptyFile() throws Exception {
+ store.storeEmptyFile("test/hadoop/file1");
+ fs.open(path("/test/hadoop/file1")).close();
+ }
+
+ public void testBlockSize() throws Exception {
+ Path file = path("/test/hadoop/file");
+ createFile(file);
+ assertEquals("Default block size", fs.getDefaultBlockSize(file),
+ fs.getFileStatus(file).getBlockSize());
+
+ // Block size is determined at read time
+ long newBlockSize = fs.getDefaultBlockSize(file) * 2;
+ fs.getConf().setLong("fs.s3n.block.size", newBlockSize);
+ assertEquals("Double default block size", newBlockSize,
+ fs.getFileStatus(file).getBlockSize());
+ }
+
+ public void testRetryOnIoException() throws Exception {
+ class TestInputStream extends InputStream {
+ boolean shouldThrow = false;
+ int throwCount = 0;
+ int pos = 0;
+ byte[] bytes;
+
+ public TestInputStream() {
+ bytes = new byte[256];
+ for (int i = 0; i < 256; i++) {
+ bytes[i] = (byte)i;
+ }
+ }
+
+ @Override
+ public int read() throws IOException {
+ shouldThrow = !shouldThrow;
+ if (shouldThrow) {
+ throwCount++;
+ throw new IOException();
+ }
+ return pos++;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ shouldThrow = !shouldThrow;
+ if (shouldThrow) {
+ throwCount++;
+ throw new IOException();
+ }
+
+ int sizeToRead = Math.min(len, 256 - pos);
+ for (int i = 0; i < sizeToRead; i++) {
+ b[i] = bytes[pos + i];
+ }
+ pos += sizeToRead;
+ return sizeToRead;
+ }
+ }
+
+ final InputStream is = new TestInputStream();
+
+ class MockNativeFileSystemStore extends Jets3tNativeFileSystemStore {
+ @Override
+ public InputStream retrieve(String key, long byteRangeStart) throws IOException {
+ return is;
+ }
+ }
+
+ NativeS3FsInputStream stream = new NativeS3FsInputStream(new MockNativeFileSystemStore(), null, is, "");
+
+ // Test reading methods.
+ byte[] result = new byte[256];
+ for (int i = 0; i < 128; i++) {
+ result[i] = (byte)stream.read();
+ }
+ for (int i = 128; i < 256; i += 8) {
+ byte[] temp = new byte[8];
+ int read = stream.read(temp, 0, 8);
+ assertEquals(8, read);
+ System.arraycopy(temp, 0, result, i, 8);
+ }
+
+ // Assert correct
+ for (int i = 0; i < 256; i++) {
+ assertEquals((byte)i, result[i]);
+ }
+
+ // Test to make sure the throw path was exercised.
+ // 144 = 128 + (128 / 8)
+ assertEquals(144, ((TestInputStream)is).throwCount);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/S3NInMemoryFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/S3NInMemoryFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/S3NInMemoryFileSystem.java
new file mode 100644
index 0000000..c0ea85b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/S3NInMemoryFileSystem.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import org.apache.hadoop.fs.s3native.NativeS3FileSystem;
+import org.apache.hadoop.fs.s3native.InMemoryNativeFileSystemStore;
+
+/**
+ * A helper implementation of {@link NativeS3FileSystem}
+ * without actually connecting to S3 for unit testing.
+ */
+public class S3NInMemoryFileSystem extends NativeS3FileSystem {
+ public S3NInMemoryFileSystem() {
+ super(new InMemoryNativeFileSystemStore());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
new file mode 100644
index 0000000..664d39e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.IOException;
+
+public class TestInMemoryNativeS3FileSystemContract
+ extends NativeS3FileSystemContractBaseTest {
+
+ @Override
+ NativeFileSystemStore getNativeFileSystemStore() throws IOException {
+ return new InMemoryNativeFileSystemStore();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
new file mode 100644
index 0000000..b1078a4
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import static org.junit.Assert.*;
+import static org.junit.Assume.*;
+
+import org.junit.Before;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.security.DigestInputStream;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+
+public class TestJets3tNativeFileSystemStore {
+ private Configuration conf;
+ private Jets3tNativeFileSystemStore store;
+ private NativeS3FileSystem fs;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ store = new Jets3tNativeFileSystemStore();
+ fs = new NativeS3FileSystem(store);
+ conf.setBoolean("fs.s3n.multipart.uploads.enabled", true);
+ conf.setLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024);
+ fs.initialize(URI.create(conf.get("test.fs.s3n.name")), conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try {
+ store.purge("test");
+ } catch (Exception e) {}
+ }
+
+ @BeforeClass
+ public static void checkSettings() throws Exception {
+ Configuration conf = new Configuration();
+ assumeNotNull(conf.get("fs.s3n.awsAccessKeyId"));
+ assumeNotNull(conf.get("fs.s3n.awsSecretAccessKey"));
+ assumeNotNull(conf.get("test.fs.s3n.name"));
+ }
+
+ protected void writeRenameReadCompare(Path path, long len)
+ throws IOException, NoSuchAlgorithmException {
+ // If len > fs.s3n.multipart.uploads.block.size,
+ // we'll use a multipart upload copy
+ MessageDigest digest = MessageDigest.getInstance("MD5");
+ OutputStream out = new BufferedOutputStream(
+ new DigestOutputStream(fs.create(path, false), digest));
+ for (long i = 0; i < len; i++) {
+ out.write('Q');
+ }
+ out.flush();
+ out.close();
+
+ assertTrue("Exists", fs.exists(path));
+
+ // Depending on if this file is over 5 GB or not,
+ // rename will cause a multipart upload copy
+ Path copyPath = path.suffix(".copy");
+ fs.rename(path, copyPath);
+
+ assertTrue("Copy exists", fs.exists(copyPath));
+
+ // Download file from S3 and compare the digest against the original
+ MessageDigest digest2 = MessageDigest.getInstance("MD5");
+ InputStream in = new BufferedInputStream(
+ new DigestInputStream(fs.open(copyPath), digest2));
+ long copyLen = 0;
+ while (in.read() != -1) {copyLen++;}
+ in.close();
+
+ assertEquals("Copy length matches original", len, copyLen);
+ assertArrayEquals("Digests match", digest.digest(), digest2.digest());
+ }
+
+ @Test
+ public void testSmallUpload() throws IOException, NoSuchAlgorithmException {
+ // Regular upload, regular copy
+ writeRenameReadCompare(new Path("/test/small"), 16384);
+ }
+
+ @Test
+ public void testMediumUpload() throws IOException, NoSuchAlgorithmException {
+ // Multipart upload, regular copy
+ writeRenameReadCompare(new Path("/test/medium"), 33554432); // 100 MB
+ }
+
+ @Test
+ public void testExtraLargeUpload()
+ throws IOException, NoSuchAlgorithmException {
+ // Multipart upload, multipart copy
+ writeRenameReadCompare(new Path("/test/xlarge"), 5368709121L); // 5GB+1byte
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3NInMemoryFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3NInMemoryFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3NInMemoryFileSystem.java
new file mode 100644
index 0000000..b457df2
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3NInMemoryFileSystem.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URI;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+
+public class TestS3NInMemoryFileSystem extends TestCase {
+
+ private static final String TEST_PATH = "s3n://test/data.txt";
+
+ private static final String TEST_DATA = "Sample data for testing.";
+
+ private S3NInMemoryFileSystem fs;
+
+ @Override
+ public void setUp() throws IOException {
+ fs = new S3NInMemoryFileSystem();
+ fs.initialize(URI.create("s3n://test/"), new Configuration());
+ }
+
+ public void testBasicReadWriteIO() throws IOException {
+ FSDataOutputStream writeData = fs.create(new Path(TEST_PATH));
+ writeData.write(TEST_DATA.getBytes());
+ writeData.flush();
+ writeData.close();
+
+ FSDataInputStream readData = fs.open(new Path(TEST_PATH));
+ BufferedReader br = new BufferedReader(new InputStreamReader(readData));
+ String line = "";
+ StringBuffer stringBuffer = new StringBuffer();
+ while ((line = br.readLine()) != null) {
+ stringBuffer.append(line);
+ }
+ br.close();
+
+ assert(TEST_DATA.equals(stringBuffer.toString()));
+ }
+
+ @Override
+ public void tearDown() throws IOException {
+ fs.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
new file mode 100644
index 0000000..ab46178
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
@@ -0,0 +1,95 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <!--
+ S3N is a blobstore, with very different behavior than a
+ classic filesystem.
+ -->
+
+ <property>
+ <name>fs.contract.test.root-tests-enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.test.random-seek-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-blobstore</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-case-sensitive</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-returns-false-if-source-missing</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-append</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-directory-delete</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-rename</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-block-locality</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-concat</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rejects-seek-past-eof</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-strict-exceptions</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-unix-permissions</name>
+ <value>false</value>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 38043f7..7a01085 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -85,6 +85,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <scope>compile</scope>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
<scope>compile</scope>
<version>${project.version}</version>
[07/20] git commit: YARN-2158. Fixed
TestRMWebServicesAppsModification#testSingleAppKill test failure. Contributed
by Varun Vasudev
Posted by sz...@apache.org.
YARN-2158. Fixed TestRMWebServicesAppsModification#testSingleAppKill test failure. Contributed by Varun Vasudev
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbfe2637
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbfe2637
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbfe2637
Branch: refs/heads/HDFS-6584
Commit: cbfe26370b85161c79fdd48bf69c95d5725d8f6a
Parents: 47bdfa0
Author: Jian He <ji...@apache.org>
Authored: Wed Sep 10 12:47:34 2014 -0700
Committer: Jian He <ji...@apache.org>
Committed: Wed Sep 10 12:47:34 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../TestRMWebServicesAppsModification.java | 50 +++++++-------------
2 files changed, 20 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbfe2637/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2feae46..cf5ec65 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -314,6 +314,9 @@ Release 2.6.0 - UNRELEASED
YARN-2459. RM crashes if App gets rejected for any reason
and HA is enabled. (Jian He via xgong)
+ YARN-2158. Fixed TestRMWebServicesAppsModification#testSingleAppKill test
+ failure. (Varun Vasudev via jianhe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbfe2637/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index e02e410..536aa8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -45,7 +45,6 @@ import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
@@ -78,7 +77,6 @@ import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -112,12 +110,12 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
private static final int CONTAINER_MB = 1024;
- private Injector injector;
+ private static Injector injector;
private String webserviceUserName = "testuser";
private boolean setAuthFilter = false;
- public class GuiceServletConfig extends GuiceServletContextListener {
+ public static class GuiceServletConfig extends GuiceServletContextListener {
@Override
protected Injector getInjector() {
@@ -263,9 +261,9 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
.constructWebResource("apps", app.getApplicationId().toString(),
"state").accept(mediaType).get(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
- if (mediaType == MediaType.APPLICATION_JSON) {
+ if (mediaType.equals(MediaType.APPLICATION_JSON)) {
verifyAppStateJson(response, RMAppState.ACCEPTED);
- } else if (mediaType == MediaType.APPLICATION_XML) {
+ } else if (mediaType.equals(MediaType.APPLICATION_XML)) {
verifyAppStateXML(response, RMAppState.ACCEPTED);
}
}
@@ -285,10 +283,6 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
amNodeManager.nodeHeartbeat(true);
- ClientResponse response =
- this
- .constructWebResource("apps", app.getApplicationId().toString(),
- "state").accept(mediaType).get(ClientResponse.class);
AppState targetState =
new AppState(YarnApplicationState.KILLED.toString());
@@ -298,7 +292,7 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
} else {
entity = targetState;
}
- response =
+ ClientResponse response =
this
.constructWebResource("apps", app.getApplicationId().toString(),
"state").entity(entity, contentType).accept(mediaType)
@@ -309,10 +303,12 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
continue;
}
assertEquals(Status.ACCEPTED, response.getClientResponseStatus());
- if (mediaType == MediaType.APPLICATION_JSON) {
- verifyAppStateJson(response, RMAppState.KILLING, RMAppState.ACCEPTED);
+ if (mediaType.equals(MediaType.APPLICATION_JSON)) {
+ verifyAppStateJson(response, RMAppState.FINAL_SAVING,
+ RMAppState.KILLED, RMAppState.KILLING, RMAppState.ACCEPTED);
} else {
- verifyAppStateXML(response, RMAppState.KILLING, RMAppState.ACCEPTED);
+ verifyAppStateXML(response, RMAppState.FINAL_SAVING,
+ RMAppState.KILLED, RMAppState.KILLING, RMAppState.ACCEPTED);
}
String locationHeaderValue =
@@ -338,7 +334,7 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
|| (response.getClientResponseStatus() == Status.OK));
if (response.getClientResponseStatus() == Status.OK) {
assertEquals(RMAppState.KILLED, app.getState());
- if (mediaType == MediaType.APPLICATION_JSON) {
+ if (mediaType.equals(MediaType.APPLICATION_JSON)) {
verifyAppStateJson(response, RMAppState.KILLED);
} else {
verifyAppStateXML(response, RMAppState.KILLED);
@@ -350,7 +346,6 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
}
rm.stop();
- return;
}
@Test
@@ -396,7 +391,6 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
}
rm.stop();
- return;
}
private static String appStateToJSON(AppState state) throws Exception {
@@ -422,7 +416,6 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
}
String msg = "app state incorrect, got " + responseState;
assertTrue(msg, valid);
- return;
}
protected static void verifyAppStateXML(ClientResponse response,
@@ -447,7 +440,6 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
}
String msg = "app state incorrect, got " + state;
assertTrue(msg, valid);
- return;
}
@Test(timeout = 30000)
@@ -487,7 +479,6 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
validateResponseStatus(response, Status.FORBIDDEN);
}
rm.stop();
- return;
}
@@ -510,7 +501,6 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
}
rm.stop();
- return;
}
@After
@@ -571,7 +561,6 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
testGetNewApplication(acceptMedia);
}
rm.stop();
- return;
}
protected String testGetNewApplication(String mediaType) throws JSONException,
@@ -606,7 +595,7 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
protected String validateGetNewApplicationJsonResponse(JSONObject json)
throws JSONException {
String appId = json.getString("application-id");
- assertTrue(appId.isEmpty() == false);
+ assertTrue(!appId.isEmpty());
JSONObject maxResources = json.getJSONObject("maximum-resource-capability");
long memory = maxResources.getLong("memory");
long vCores = maxResources.getLong("vCores");
@@ -626,7 +615,7 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
assertEquals("incorrect number of elements", 1, nodes.getLength());
Element element = (Element) nodes.item(0);
String appId = WebServicesTestUtils.getXmlString(element, "application-id");
- assertTrue(appId.isEmpty() == false);
+ assertTrue(!appId.isEmpty());
NodeList maxResourceNodes =
element.getElementsByTagName("maximum-resource-capability");
assertEquals(1, maxResourceNodes.getLength());
@@ -656,7 +645,6 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
}
}
rm.stop();
- return;
}
public void testAppSubmit(String acceptMedia, String contentMedia)
@@ -721,14 +709,14 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
this.constructWebResource(urlPath).accept(acceptMedia)
.entity(appInfo, contentMedia).post(ClientResponse.class);
- if (this.isAuthenticationEnabled() == false) {
+ if (!this.isAuthenticationEnabled()) {
assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus());
return;
}
assertEquals(Status.ACCEPTED, response.getClientResponseStatus());
- assertTrue(response.getHeaders().getFirst(HttpHeaders.LOCATION).isEmpty() == false);
+ assertTrue(!response.getHeaders().getFirst(HttpHeaders.LOCATION).isEmpty());
String locURL = response.getHeaders().getFirst(HttpHeaders.LOCATION);
- assertTrue(locURL.indexOf("/apps/application") != -1);
+ assertTrue(locURL.contains("/apps/application"));
appId = locURL.substring(locURL.indexOf("/apps/") + "/apps/".length());
WebResource res = resource().uri(new URI(locURL));
@@ -775,7 +763,6 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
this.constructWebResource("apps", appId).accept(acceptMedia)
.get(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
- return;
}
public void testAppSubmitErrors(String acceptMedia, String contentMedia)
@@ -785,14 +772,13 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
// REST API and make sure we get the right error response codes
String urlPath = "apps";
- String appId = "";
ApplicationSubmissionContextInfo appInfo = new ApplicationSubmissionContextInfo();
ClientResponse response =
this.constructWebResource(urlPath).accept(acceptMedia)
.entity(appInfo, contentMedia).post(ClientResponse.class);
validateResponseStatus(response, Status.BAD_REQUEST);
- appId = "random";
+ String appId = "random";
appInfo.setApplicationId(appId);
response =
this.constructWebResource(urlPath).accept(acceptMedia)
@@ -827,8 +813,6 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
this.constructWebResource(urlPath).accept(acceptMedia)
.entity(appInfo, contentMedia).post(ClientResponse.class);
validateResponseStatus(response, Status.BAD_REQUEST);
-
- return;
}
@Test
[19/20] git commit: YARN-2440. Enabled Nodemanagers to limit the
aggregate cpu usage across all containers to a preconfigured limit.
Contributed by Varun Vasudev.
Posted by sz...@apache.org.
YARN-2440. Enabled Nodemanagers to limit the aggregate cpu usage across all containers to a preconfigured limit. Contributed by Varun Vasudev.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4be95175
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4be95175
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4be95175
Branch: refs/heads/HDFS-6584
Commit: 4be95175cdb58ff12a27ab443d609d3b46da7bfa
Parents: 83be3ad
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Wed Sep 10 19:22:52 2014 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Wed Sep 10 19:22:52 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 6 +
.../src/main/resources/yarn-default.xml | 16 +-
.../util/CgroupsLCEResourcesHandler.java | 94 +++++++++++-
.../util/NodeManagerHardwareUtils.java | 79 ++++++++++
.../util/TestCgroupsLCEResourcesHandler.java | 147 ++++++++++++++++++-
.../util/TestNodeManagerHardwareUtils.java | 72 +++++++++
7 files changed, 408 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4be95175/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b28f8f8..6a871a5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -70,6 +70,9 @@ Release 2.6.0 - UNRELEASED
YARN-415. Capture aggregate memory allocation at the app-level for chargeback.
(Eric Payne & Andrey Klochkov via jianhe)
+ YARN-2440. Enabled Nodemanagers to limit the aggregate cpu usage across all
+ containers to a preconfigured limit. (Varun Vasudev via vinodkv)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4be95175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7b7511d..7c71a17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -723,6 +723,12 @@ public class YarnConfiguration extends Configuration {
/** Number of Virtual CPU Cores which can be allocated for containers.*/
public static final String NM_VCORES = NM_PREFIX + "resource.cpu-vcores";
public static final int DEFAULT_NM_VCORES = 8;
+
+ /** Percentage of overall CPU which can be allocated for containers. */
+ public static final String NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT =
+ NM_PREFIX + "resource.percentage-physical-cpu-limit";
+ public static final int DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT =
+ 100;
/** NM Webapp address.**/
public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4be95175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 9b4a90f..04e458c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -871,13 +871,25 @@
</property>
<property>
- <description>Number of CPU cores that can be allocated
- for containers.</description>
+ <description>Number of vcores that can be allocated
+ for containers. This is used by the RM scheduler when allocating
+ resources for containers. This is not used to limit the number of
+ physical cores used by YARN containers.</description>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>8</value>
</property>
<property>
+ <description>Percentage of CPU that can be allocated
+ for containers. This setting allows users to limit the amount of
+ CPU that YARN containers use. Currently functional only
+ on Linux using cgroups. The default is to use 100% of CPU.
+ </description>
+ <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
+ <value>100</value>
+ </property>
+
+ <property>
<description>NM Webapp address.</description>
<name>yarn.nodemanager.webapp.address</name>
<value>${yarn.nodemanager.hostname}:8042</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4be95175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index d5bd225..0b6c2ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -33,6 +33,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -42,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.SystemClock;
public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
@@ -59,7 +61,11 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
private final String MTAB_FILE = "/proc/mounts";
private final String CGROUPS_FSTYPE = "cgroup";
private final String CONTROLLER_CPU = "cpu";
+ private final String CPU_PERIOD_US = "cfs_period_us";
+ private final String CPU_QUOTA_US = "cfs_quota_us";
private final int CPU_DEFAULT_WEIGHT = 1024; // set by kernel
+ private final int MAX_QUOTA_US = 1000 * 1000;
+ private final int MIN_PERIOD_US = 1000;
private final Map<String, String> controllerPaths; // Controller -> path
private long deleteCgroupTimeout;
@@ -106,8 +112,15 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
}
public void init(LinuxContainerExecutor lce) throws IOException {
+ this.init(lce,
+ ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf));
+ }
+
+ @VisibleForTesting
+ void init(LinuxContainerExecutor lce, ResourceCalculatorPlugin plugin)
+ throws IOException {
initConfig();
-
+
// mount cgroups if requested
if (cgroupMount && cgroupMountPath != null) {
ArrayList<String> cgroupKVs = new ArrayList<String>();
@@ -117,8 +130,74 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
}
initializeControllerPaths();
+
+ // cap overall usage to the number of cores allocated to YARN
+ float yarnProcessors =
+ NodeManagerHardwareUtils.getContainersCores(plugin, conf);
+ int systemProcessors = plugin.getNumProcessors();
+ if (systemProcessors != (int) yarnProcessors) {
+ LOG.info("YARN containers restricted to " + yarnProcessors + " cores");
+ int[] limits = getOverallLimits(yarnProcessors);
+ updateCgroup(CONTROLLER_CPU, "", CPU_PERIOD_US, String.valueOf(limits[0]));
+ updateCgroup(CONTROLLER_CPU, "", CPU_QUOTA_US, String.valueOf(limits[1]));
+ } else if (cpuLimitsExist()) {
+ LOG.info("Removing CPU constraints for YARN containers.");
+ updateCgroup(CONTROLLER_CPU, "", CPU_QUOTA_US, String.valueOf(-1));
+ }
}
+ boolean cpuLimitsExist() throws IOException {
+ String path = pathForCgroup(CONTROLLER_CPU, "");
+ File quotaFile = new File(path, CONTROLLER_CPU + "." + CPU_QUOTA_US);
+ if (quotaFile.exists()) {
+ String contents = FileUtils.readFileToString(quotaFile, "UTF-8");
+ int quotaUS = Integer.parseInt(contents.trim());
+ if (quotaUS != -1) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @VisibleForTesting
+ int[] getOverallLimits(float yarnProcessors) {
+
+ int[] ret = new int[2];
+
+ if (yarnProcessors < 0.01f) {
+ throw new IllegalArgumentException("Number of processors can't be <= 0.");
+ }
+
+ int quotaUS = MAX_QUOTA_US;
+ int periodUS = (int) (MAX_QUOTA_US / yarnProcessors);
+ if (yarnProcessors < 1.0f) {
+ periodUS = MAX_QUOTA_US;
+ quotaUS = (int) (periodUS * yarnProcessors);
+ if (quotaUS < MIN_PERIOD_US) {
+ LOG
+ .warn("The quota calculated for the cgroup was too low. The minimum value is "
+ + MIN_PERIOD_US + ", calculated value is " + quotaUS
+ + ". Setting quota to minimum value.");
+ quotaUS = MIN_PERIOD_US;
+ }
+ }
+
+ // cfs_period_us can't be less than 1000 microseconds
+ // if the value of periodUS is less than 1000, we can't really use cgroups
+ // to limit cpu
+ if (periodUS < MIN_PERIOD_US) {
+ LOG
+ .warn("The period calculated for the cgroup was too low. The minimum value is "
+ + MIN_PERIOD_US + ", calculated value is " + periodUS
+ + ". Using all available CPU.");
+ periodUS = MAX_QUOTA_US;
+ quotaUS = -1;
+ }
+
+ ret[0] = periodUS;
+ ret[1] = quotaUS;
+ return ret;
+ }
boolean isCpuWeightEnabled() {
return this.cpuWeightEnabled;
@@ -274,7 +353,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
BufferedReader in = null;
try {
- in = new BufferedReader(new FileReader(new File(MTAB_FILE)));
+ in = new BufferedReader(new FileReader(new File(getMtabFileName())));
for (String str = in.readLine(); str != null;
str = in.readLine()) {
@@ -292,13 +371,13 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
}
}
} catch (IOException e) {
- throw new IOException("Error while reading " + MTAB_FILE, e);
+ throw new IOException("Error while reading " + getMtabFileName(), e);
} finally {
// Close the streams
try {
in.close();
} catch (IOException e2) {
- LOG.warn("Error closing the stream: " + MTAB_FILE, e2);
+ LOG.warn("Error closing the stream: " + getMtabFileName(), e2);
}
}
@@ -334,7 +413,12 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
}
} else {
throw new IOException("Not able to enforce cpu weights; cannot find "
- + "cgroup for cpu controller in " + MTAB_FILE);
+ + "cgroup for cpu controller in " + getMtabFileName());
}
}
+
+ @VisibleForTesting
+ String getMtabFileName() {
+ return MTAB_FILE;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4be95175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java
new file mode 100644
index 0000000..07cf698
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class NodeManagerHardwareUtils {
+
+ /**
+ *
+ * Returns the fraction of CPU cores that should be used for YARN containers.
+ * The number is derived based on various configuration params such as
+ * YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
+ *
+ * @param conf
+ * - Configuration object
+ * @return Fraction of CPU cores to be used for YARN containers
+ */
+ public static float getContainersCores(Configuration conf) {
+ ResourceCalculatorPlugin plugin =
+ ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf);
+ return NodeManagerHardwareUtils.getContainersCores(plugin, conf);
+ }
+
+ /**
+ *
+ * Returns the fraction of CPU cores that should be used for YARN containers.
+ * The number is derived based on various configuration params such as
+ * YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
+ *
+ * @param plugin
+ * - ResourceCalculatorPlugin object to determine hardware specs
+ * @param conf
+ * - Configuration object
+ * @return Fraction of CPU cores to be used for YARN containers
+ */
+ public static float getContainersCores(ResourceCalculatorPlugin plugin,
+ Configuration conf) {
+ int numProcessors = plugin.getNumProcessors();
+ int nodeCpuPercentage =
+ Math.min(conf.getInt(
+ YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT,
+ YarnConfiguration.DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT),
+ 100);
+ nodeCpuPercentage = Math.max(0, nodeCpuPercentage);
+
+ if (nodeCpuPercentage == 0) {
+ String message =
+ "Illegal value for "
+ + YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
+ + ". Value cannot be less than or equal to 0.";
+ throw new IllegalArgumentException(message);
+ }
+
+ return (nodeCpuPercentage * numProcessors) / 100.0f;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4be95175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
index 611045e..4506898 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
@@ -17,13 +17,18 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.util;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.junit.Assert;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.Clock;
import org.junit.Test;
+import org.mockito.Mockito;
-import java.io.File;
-import java.io.FileOutputStream;
+import java.io.*;
+import java.util.List;
+import java.util.Scanner;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
@@ -70,4 +75,142 @@ public class TestCgroupsLCEResourcesHandler {
Assert.assertFalse(handler.deleteCgroup(file.getPath()));
}
+ static class MockLinuxContainerExecutor extends LinuxContainerExecutor {
+ @Override
+ public void mountCgroups(List<String> x, String y) {
+ }
+ }
+
+ static class CustomCgroupsLCEResourceHandler extends
+ CgroupsLCEResourcesHandler {
+
+ String mtabFile;
+ int[] limits = new int[2];
+
+ @Override
+ int[] getOverallLimits(float x) {
+ return limits;
+ }
+
+ void setMtabFile(String file) {
+ mtabFile = file;
+ }
+
+ @Override
+ String getMtabFileName() {
+ return mtabFile;
+ }
+ }
+
+ @Test
+ public void testInit() throws IOException {
+ LinuxContainerExecutor mockLCE = new MockLinuxContainerExecutor();
+ CustomCgroupsLCEResourceHandler handler =
+ new CustomCgroupsLCEResourceHandler();
+ YarnConfiguration conf = new YarnConfiguration();
+ final int numProcessors = 4;
+ ResourceCalculatorPlugin plugin =
+ Mockito.mock(ResourceCalculatorPlugin.class);
+ Mockito.doReturn(numProcessors).when(plugin).getNumProcessors();
+ handler.setConf(conf);
+ handler.initConfig();
+
+ // create mock cgroup
+ File cgroupDir = new File("target", UUID.randomUUID().toString());
+ if (!cgroupDir.mkdir()) {
+ String message = "Could not create dir " + cgroupDir.getAbsolutePath();
+ throw new IOException(message);
+ }
+ File cgroupMountDir = new File(cgroupDir.getAbsolutePath(), "hadoop-yarn");
+ if (!cgroupMountDir.mkdir()) {
+ String message =
+ "Could not create dir " + cgroupMountDir.getAbsolutePath();
+ throw new IOException(message);
+ }
+
+ // create mock mtab
+ String mtabContent =
+ "none " + cgroupDir.getAbsolutePath() + " cgroup rw,relatime,cpu 0 0";
+ File mockMtab = new File("target", UUID.randomUUID().toString());
+ if (!mockMtab.exists()) {
+ if (!mockMtab.createNewFile()) {
+ String message = "Could not create file " + mockMtab.getAbsolutePath();
+ throw new IOException(message);
+ }
+ }
+ FileWriter mtabWriter = new FileWriter(mockMtab.getAbsoluteFile());
+ mtabWriter.write(mtabContent);
+ mtabWriter.close();
+ mockMtab.deleteOnExit();
+
+ // setup our handler and call init()
+ handler.setMtabFile(mockMtab.getAbsolutePath());
+
+ // check values
+ // in this case, we're using all cpu so the files
+ // shouldn't exist(because init won't create them
+ handler.init(mockLCE, plugin);
+ File periodFile = new File(cgroupMountDir, "cpu.cfs_period_us");
+ File quotaFile = new File(cgroupMountDir, "cpu.cfs_quota_us");
+ Assert.assertFalse(periodFile.exists());
+ Assert.assertFalse(quotaFile.exists());
+
+ // subset of cpu being used, files should be created
+ conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 75);
+ handler.limits[0] = 100 * 1000;
+ handler.limits[1] = 1000 * 1000;
+ handler.init(mockLCE, plugin);
+ int period = readIntFromFile(periodFile);
+ int quota = readIntFromFile(quotaFile);
+ Assert.assertEquals(100 * 1000, period);
+ Assert.assertEquals(1000 * 1000, quota);
+
+ // set cpu back to 100, quota should be -1
+ conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 100);
+ handler.limits[0] = 100 * 1000;
+ handler.limits[1] = 1000 * 1000;
+ handler.init(mockLCE, plugin);
+ quota = readIntFromFile(quotaFile);
+ Assert.assertEquals(-1, quota);
+
+ FileUtils.deleteQuietly(cgroupDir);
+ }
+
+ private int readIntFromFile(File targetFile) throws IOException {
+ Scanner scanner = new Scanner(targetFile);
+ if (scanner.hasNextInt()) {
+ return scanner.nextInt();
+ }
+ return -1;
+ }
+
+ @Test
+ public void testGetOverallLimits() {
+
+ int expectedQuota = 1000 * 1000;
+ CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler();
+
+ int[] ret = handler.getOverallLimits(2);
+ Assert.assertEquals(expectedQuota / 2, ret[0]);
+ Assert.assertEquals(expectedQuota, ret[1]);
+
+ ret = handler.getOverallLimits(2000);
+ Assert.assertEquals(expectedQuota, ret[0]);
+ Assert.assertEquals(-1, ret[1]);
+
+ int[] params = { 0, -1 };
+ for (int cores : params) {
+ try {
+ handler.getOverallLimits(cores);
+ Assert.fail("Function call should throw error.");
+ } catch (IllegalArgumentException ie) {
+ // expected
+ }
+ }
+
+ // test minimums
+ ret = handler.getOverallLimits(1000 * 1000);
+ Assert.assertEquals(1000 * 1000, ret[0]);
+ Assert.assertEquals(-1, ret[1]);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4be95175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
new file mode 100644
index 0000000..e1af948
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.util;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestNodeManagerHardwareUtils {
+
+ @Test
+ public void testGetContainerCores() {
+
+ YarnConfiguration conf = new YarnConfiguration();
+ float ret;
+ final int numProcessors = 4;
+ ResourceCalculatorPlugin plugin =
+ Mockito.mock(ResourceCalculatorPlugin.class);
+ Mockito.doReturn(numProcessors).when(plugin).getNumProcessors();
+
+ conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 0);
+ try {
+ NodeManagerHardwareUtils.getContainersCores(plugin, conf);
+ Assert.fail("getContainerCores should have thrown exception");
+ } catch (IllegalArgumentException ie) {
+ // expected
+ }
+
+ conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT,
+ 100);
+ ret = NodeManagerHardwareUtils.getContainersCores(plugin, conf);
+ Assert.assertEquals(4, (int) ret);
+
+ conf
+ .setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 50);
+ ret = NodeManagerHardwareUtils.getContainersCores(plugin, conf);
+ Assert.assertEquals(2, (int) ret);
+
+ conf
+ .setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 75);
+ ret = NodeManagerHardwareUtils.getContainersCores(plugin, conf);
+ Assert.assertEquals(3, (int) ret);
+
+ conf
+ .setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 85);
+ ret = NodeManagerHardwareUtils.getContainersCores(plugin, conf);
+ Assert.assertEquals(3.4, ret, 0.1);
+
+ conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT,
+ 110);
+ ret = NodeManagerHardwareUtils.getContainersCores(plugin, conf);
+ Assert.assertEquals(4, (int) ret);
+ }
+}
[18/20] git commit: YARN-415. Capture aggregate memory allocation at
the app-level for chargeback. Contributed by Eric Payne & Andrey Klochkov
Posted by sz...@apache.org.
YARN-415. Capture aggregate memory allocation at the app-level for chargeback. Contributed by Eric Payne & Andrey Klochkov
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83be3ad4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83be3ad4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83be3ad4
Branch: refs/heads/HDFS-6584
Commit: 83be3ad44484bf8a24cb90de4b9c26ab59d226a8
Parents: 5ec7fcd
Author: Jian He <ji...@apache.org>
Authored: Wed Sep 10 18:19:53 2014 -0700
Committer: Jian He <ji...@apache.org>
Committed: Wed Sep 10 18:20:54 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/ApplicationResourceUsageReport.java | 41 +-
.../src/main/proto/yarn_protos.proto | 2 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 6 +
.../hadoop/yarn/client/cli/TestYarnCLI.java | 8 +-
.../ApplicationResourceUsageReportPBImpl.java | 24 ++
.../hadoop/yarn/server/utils/BuilderUtils.java | 5 +-
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../recovery/FileSystemRMStateStore.java | 4 +-
.../recovery/MemoryRMStateStore.java | 9 +-
.../resourcemanager/recovery/RMStateStore.java | 24 +-
.../recovery/ZKRMStateStore.java | 4 +-
.../records/ApplicationAttemptStateData.java | 32 +-
.../pb/ApplicationAttemptStateDataPBImpl.java | 24 ++
.../server/resourcemanager/rmapp/RMAppImpl.java | 17 +-
.../resourcemanager/rmapp/RMAppMetrics.java | 15 +-
.../attempt/AggregateAppResourceUsage.java | 60 +++
.../rmapp/attempt/RMAppAttemptImpl.java | 16 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 36 +-
.../rmcontainer/RMContainerImpl.java | 26 +-
.../scheduler/SchedulerApplicationAttempt.java | 41 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 3 +
.../scheduler/fair/FSAppAttempt.java | 3 +
.../server/resourcemanager/webapp/AppBlock.java | 5 +-
.../resourcemanager/webapp/dao/AppInfo.java | 12 +
.../yarn_server_resourcemanager_recovery.proto | 4 +-
.../resourcemanager/TestClientRMService.java | 60 ++-
.../TestContainerResourceUsage.java | 401 +++++++++++++++++++
.../applicationsmanager/MockAsm.java | 19 +-
.../recovery/RMStateStoreTestBase.java | 18 +-
.../rmapp/TestRMAppTransitions.java | 16 +-
.../attempt/TestRMAppAttemptTransitions.java | 72 +++-
.../rmcontainer/TestRMContainerImpl.java | 11 +
.../scheduler/capacity/TestLeafQueue.java | 89 ++--
.../scheduler/fair/FairSchedulerTestBase.java | 4 +-
.../webapp/TestRMWebAppFairScheduler.java | 2 +-
.../webapp/TestRMWebServicesApps.java | 2 +-
.../src/site/apt/ResourceManagerRest.apt.vm | 22 +-
38 files changed, 1045 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7968071..b28f8f8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -67,6 +67,9 @@ Release 2.6.0 - UNRELEASED
YARN-2394. FairScheduler: Configure fairSharePreemptionThreshold per queue.
(Wei Yan via kasha)
+ YARN-415. Capture aggregate memory allocation at the app-level for chargeback.
+ (Eric Payne & Andrey Klochkov via jianhe)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index 6e9c76f..b20d832 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -35,7 +35,8 @@ public abstract class ApplicationResourceUsageReport {
@Unstable
public static ApplicationResourceUsageReport newInstance(
int numUsedContainers, int numReservedContainers, Resource usedResources,
- Resource reservedResources, Resource neededResources) {
+ Resource reservedResources, Resource neededResources, long memorySeconds,
+ long vcoreSeconds) {
ApplicationResourceUsageReport report =
Records.newRecord(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
@@ -43,6 +44,8 @@ public abstract class ApplicationResourceUsageReport {
report.setUsedResources(usedResources);
report.setReservedResources(reservedResources);
report.setNeededResources(neededResources);
+ report.setMemorySeconds(memorySeconds);
+ report.setVcoreSeconds(vcoreSeconds);
return report;
}
@@ -113,4 +116,40 @@ public abstract class ApplicationResourceUsageReport {
@Private
@Unstable
public abstract void setNeededResources(Resource needed_resources);
+
+ /**
+ * Set the aggregated amount of memory (in megabytes) the application has
+ * allocated times the number of seconds the application has been running.
+ * @param memory_seconds the aggregated amount of memory seconds
+ */
+ @Private
+ @Unstable
+ public abstract void setMemorySeconds(long memory_seconds);
+
+ /**
+ * Get the aggregated amount of memory (in megabytes) the application has
+ * allocated times the number of seconds the application has been running.
+ * @return the aggregated amount of memory seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getMemorySeconds();
+
+ /**
+ * Set the aggregated number of vcores that the application has allocated
+ * times the number of seconds the application has been running.
+ * @param vcore_seconds the aggregated number of vcore seconds
+ */
+ @Private
+ @Unstable
+ public abstract void setVcoreSeconds(long vcore_seconds);
+
+ /**
+ * Get the aggregated number of vcores that the application has allocated
+ * times the number of seconds the application has been running.
+ * @return the aggregated number of vcore seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getVcoreSeconds();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 3f1fa6c..7e7f21b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -167,6 +167,8 @@ message ApplicationResourceUsageReportProto {
optional ResourceProto used_resources = 3;
optional ResourceProto reserved_resources = 4;
optional ResourceProto needed_resources = 5;
+ optional int64 memory_seconds = 6;
+ optional int64 vcore_seconds = 7;
}
message ApplicationReportProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 7d61a23..54cfe91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
@@ -460,6 +461,11 @@ public class ApplicationCLI extends YarnCLI {
appReportStr.println(appReport.getRpcPort());
appReportStr.print("\tAM Host : ");
appReportStr.println(appReport.getHost());
+ appReportStr.print("\tAggregate Resource Allocation : ");
+
+ ApplicationResourceUsageReport usageReport = appReport.getApplicationResourceUsageReport();
+ appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, ");
+ appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds");
appReportStr.print("\tDiagnostics : ");
appReportStr.print(appReport.getDiagnostics());
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index b408b61..47fa5ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -87,11 +88,15 @@ public class TestYarnCLI {
public void testGetApplicationReport() throws Exception {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ ApplicationResourceUsageReport usageReport =
+ ApplicationResourceUsageReport.newInstance(
+ 2, 0, null, null, null, 123456, 4567);
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
+ null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
int result = cli.run(new String[] { "application", "-status", applicationId.toString() });
@@ -113,6 +118,7 @@ public class TestYarnCLI {
pw.println("\tTracking-URL : N/A");
pw.println("\tRPC Port : 124");
pw.println("\tAM Host : host");
+ pw.println("\tAggregate Resource Allocation : 123456 MB-seconds, 4567 vcore-seconds");
pw.println("\tDiagnostics : diagnostics");
pw.close();
String appReportStr = baos.toString("UTF-8");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
index ada7165..1cabadd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
@@ -200,6 +200,30 @@ extends ApplicationResourceUsageReport {
this.neededResources = reserved_resources;
}
+ @Override
+ public synchronized void setMemorySeconds(long memory_seconds) {
+ maybeInitBuilder();
+ builder.setMemorySeconds(memory_seconds);
+ }
+
+ @Override
+ public synchronized long getMemorySeconds() {
+ ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getMemorySeconds();
+ }
+
+ @Override
+ public synchronized void setVcoreSeconds(long vcore_seconds) {
+ maybeInitBuilder();
+ builder.setVcoreSeconds(vcore_seconds);
+ }
+
+ @Override
+ public synchronized long getVcoreSeconds() {
+ ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getVcoreSeconds());
+ }
+
private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
return new ResourcePBImpl(p);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 64eb428..0cfd911 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -370,7 +370,8 @@ public class BuilderUtils {
public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
int numUsedContainers, int numReservedContainers, Resource usedResources,
- Resource reservedResources, Resource neededResources) {
+ Resource reservedResources, Resource neededResources, long memorySeconds,
+ long vcoreSeconds) {
ApplicationResourceUsageReport report =
recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
@@ -378,6 +379,8 @@ public class BuilderUtils {
report.setUsedResources(usedResources);
report.setReservedResources(reservedResources);
report.setNeededResources(neededResources);
+ report.setMemorySeconds(memorySeconds);
+ report.setVcoreSeconds(vcoreSeconds);
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index d93c45d..29c5953 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -236,5 +236,5 @@ public class RMServerUtils {
DUMMY_APPLICATION_RESOURCE_USAGE_REPORT =
BuilderUtils.newApplicationResourceUsageReport(-1, -1,
Resources.createResource(-1, -1), Resources.createResource(-1, -1),
- Resources.createResource(-1, -1));
+ Resources.createResource(-1, -1), 0, 0);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 0a3b269..4c01a61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -280,7 +280,9 @@ public class FileSystemRMStateStore extends RMStateStore {
attemptStateData.getFinalTrackingUrl(),
attemptStateData.getDiagnostics(),
attemptStateData.getFinalApplicationStatus(),
- attemptStateData.getAMContainerExitStatus());
+ attemptStateData.getAMContainerExitStatus(),
+ attemptStateData.getMemorySeconds(),
+ attemptStateData.getVcoreSeconds());
// assert child node name is same as application attempt id
assert attemptId.equals(attemptState.getAttemptId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
index f56517c..efaa039 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
@@ -138,7 +138,10 @@ public class MemoryRMStateStore extends RMStateStore {
ApplicationAttemptState attemptState =
new ApplicationAttemptState(appAttemptId,
attemptStateData.getMasterContainer(), credentials,
- attemptStateData.getStartTime());
+ attemptStateData.getStartTime(),
+ attemptStateData.getMemorySeconds(),
+ attemptStateData.getVcoreSeconds());
+
ApplicationState appState = state.getApplicationState().get(
attemptState.getAttemptId().getApplicationId());
@@ -167,7 +170,9 @@ public class MemoryRMStateStore extends RMStateStore {
attemptStateData.getFinalTrackingUrl(),
attemptStateData.getDiagnostics(),
attemptStateData.getFinalApplicationStatus(),
- attemptStateData.getAMContainerExitStatus());
+ attemptStateData.getAMContainerExitStatus(),
+ attemptStateData.getMemorySeconds(),
+ attemptStateData.getVcoreSeconds());
ApplicationState appState =
state.getApplicationState().get(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 714a108..3074d33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
@@ -265,19 +266,21 @@ public abstract class RMStateStore extends AbstractService {
String diagnostics;
int exitStatus = ContainerExitStatus.INVALID;
FinalApplicationStatus amUnregisteredFinalStatus;
+ long memorySeconds;
+ long vcoreSeconds;
public ApplicationAttemptState(ApplicationAttemptId attemptId,
Container masterContainer, Credentials appAttemptCredentials,
- long startTime) {
+ long startTime, long memorySeconds, long vcoreSeconds) {
this(attemptId, masterContainer, appAttemptCredentials, startTime, null,
- null, "", null, ContainerExitStatus.INVALID);
+ null, "", null, ContainerExitStatus.INVALID, memorySeconds, vcoreSeconds);
}
public ApplicationAttemptState(ApplicationAttemptId attemptId,
Container masterContainer, Credentials appAttemptCredentials,
long startTime, RMAppAttemptState state, String finalTrackingUrl,
String diagnostics, FinalApplicationStatus amUnregisteredFinalStatus,
- int exitStatus) {
+ int exitStatus, long memorySeconds, long vcoreSeconds) {
this.attemptId = attemptId;
this.masterContainer = masterContainer;
this.appAttemptCredentials = appAttemptCredentials;
@@ -287,6 +290,8 @@ public abstract class RMStateStore extends AbstractService {
this.diagnostics = diagnostics == null ? "" : diagnostics;
this.amUnregisteredFinalStatus = amUnregisteredFinalStatus;
this.exitStatus = exitStatus;
+ this.memorySeconds = memorySeconds;
+ this.vcoreSeconds = vcoreSeconds;
}
public Container getMasterContainer() {
@@ -316,6 +321,12 @@ public abstract class RMStateStore extends AbstractService {
public int getAMContainerExitStatus(){
return this.exitStatus;
}
+ public long getMemorySeconds() {
+ return memorySeconds;
+ }
+ public long getVcoreSeconds() {
+ return vcoreSeconds;
+ }
}
/**
@@ -587,10 +598,13 @@ public abstract class RMStateStore extends AbstractService {
public synchronized void storeNewApplicationAttempt(RMAppAttempt appAttempt) {
Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
+ AggregateAppResourceUsage resUsage =
+ appAttempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
ApplicationAttemptState attemptState =
new ApplicationAttemptState(appAttempt.getAppAttemptId(),
appAttempt.getMasterContainer(), credentials,
- appAttempt.getStartTime());
+ appAttempt.getStartTime(), resUsage.getMemorySeconds(),
+ resUsage.getVcoreSeconds());
dispatcher.getEventHandler().handle(
new RMStateStoreAppAttemptEvent(attemptState));
@@ -746,7 +760,7 @@ public abstract class RMStateStore extends AbstractService {
ApplicationAttemptState attemptState =
new ApplicationAttemptState(appAttempt.getAppAttemptId(),
appAttempt.getMasterContainer(), credentials,
- appAttempt.getStartTime());
+ appAttempt.getStartTime(), 0, 0);
appState.attempts.put(attemptState.getAttemptId(), attemptState);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index 1b1ec76..25f3819 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -603,7 +603,9 @@ public class ZKRMStateStore extends RMStateStore {
attemptStateData.getFinalTrackingUrl(),
attemptStateData.getDiagnostics(),
attemptStateData.getFinalApplicationStatus(),
- attemptStateData.getAMContainerExitStatus());
+ attemptStateData.getAMContainerExitStatus(),
+ attemptStateData.getMemorySeconds(),
+ attemptStateData.getVcoreSeconds());
appState.attempts.put(attemptState.getAttemptId(), attemptState);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
index 5cb9787..ad8cdae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
@@ -43,7 +43,8 @@ public abstract class ApplicationAttemptStateData {
ApplicationAttemptId attemptId, Container container,
ByteBuffer attemptTokens, long startTime, RMAppAttemptState finalState,
String finalTrackingUrl, String diagnostics,
- FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus) {
+ FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus,
+ long memorySeconds, long vcoreSeconds) {
ApplicationAttemptStateData attemptStateData =
Records.newRecord(ApplicationAttemptStateData.class);
attemptStateData.setAttemptId(attemptId);
@@ -55,6 +56,8 @@ public abstract class ApplicationAttemptStateData {
attemptStateData.setStartTime(startTime);
attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus);
attemptStateData.setAMContainerExitStatus(exitStatus);
+ attemptStateData.setMemorySeconds(memorySeconds);
+ attemptStateData.setVcoreSeconds(vcoreSeconds);
return attemptStateData;
}
@@ -72,7 +75,8 @@ public abstract class ApplicationAttemptStateData {
attemptState.getStartTime(), attemptState.getState(),
attemptState.getFinalTrackingUrl(), attemptState.getDiagnostics(),
attemptState.getFinalApplicationStatus(),
- attemptState.getAMContainerExitStatus());
+ attemptState.getAMContainerExitStatus(),
+ attemptState.getMemorySeconds(), attemptState.getVcoreSeconds());
}
public abstract ApplicationAttemptStateDataProto getProto();
@@ -157,4 +161,28 @@ public abstract class ApplicationAttemptStateData {
public abstract int getAMContainerExitStatus();
public abstract void setAMContainerExitStatus(int exitStatus);
+
+ /**
+ * Get the <em>memory seconds</em> (in MB seconds) of the application.
+ * @return <em>memory seconds</em> (in MB seconds) of the application
+ */
+ @Public
+ @Unstable
+ public abstract long getMemorySeconds();
+
+ @Public
+ @Unstable
+ public abstract void setMemorySeconds(long memorySeconds);
+
+ /**
+ * Get the <em>vcore seconds</em> of the application.
+ * @return <em>vcore seconds</em> of the application
+ */
+ @Public
+ @Unstable
+ public abstract long getVcoreSeconds();
+
+ @Public
+ @Unstable
+ public abstract void setVcoreSeconds(long vcoreSeconds);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
index 5c62d63..4d6212d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
@@ -229,6 +229,30 @@ public class ApplicationAttemptStateDataPBImpl extends
}
@Override
+ public long getMemorySeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getMemorySeconds();
+ }
+
+ @Override
+ public long getVcoreSeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getVcoreSeconds();
+ }
+
+ @Override
+ public void setMemorySeconds(long memorySeconds) {
+ maybeInitBuilder();
+ builder.setMemorySeconds(memorySeconds);
+ }
+
+ @Override
+ public void setVcoreSeconds(long vcoreSeconds) {
+ maybeInitBuilder();
+ builder.setVcoreSeconds(vcoreSeconds);
+ }
+
+ @Override
public FinalApplicationStatus getFinalApplicationStatus() {
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasFinalApplicationStatus()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 0b81f96..5b6df00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.Appli
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent.RMAppNodeUpdateType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
@@ -561,6 +562,10 @@ public class RMAppImpl implements RMApp, Recoverable {
}
}
}
+
+ RMAppMetrics rmAppMetrics = getRMAppMetrics();
+ appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds());
+ appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
}
if (currentApplicationAttemptId == null) {
@@ -1117,7 +1122,6 @@ public class RMAppImpl implements RMApp, Recoverable {
@Override
public RMAppState transition(RMAppImpl app, RMAppEvent event) {
-
if (!app.submissionContext.getUnmanagedAM()
&& app.getNumFailedAppAttempts() < app.maxAppAttempts) {
boolean transferStateFromPreviousAttempt = false;
@@ -1199,6 +1203,8 @@ public class RMAppImpl implements RMApp, Recoverable {
Resource resourcePreempted = Resource.newInstance(0, 0);
int numAMContainerPreempted = 0;
int numNonAMContainerPreempted = 0;
+ long memorySeconds = 0;
+ long vcoreSeconds = 0;
for (RMAppAttempt attempt : attempts.values()) {
if (null != attempt) {
RMAppAttemptMetrics attemptMetrics =
@@ -1208,10 +1214,17 @@ public class RMAppImpl implements RMApp, Recoverable {
numAMContainerPreempted += attemptMetrics.getIsPreempted() ? 1 : 0;
numNonAMContainerPreempted +=
attemptMetrics.getNumNonAMContainersPreempted();
+ // getAggregateAppResourceUsage() will calculate resource usage stats
+ // for both running and finished containers.
+ AggregateAppResourceUsage resUsage =
+ attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
+ memorySeconds += resUsage.getMemorySeconds();
+ vcoreSeconds += resUsage.getVcoreSeconds();
}
}
return new RMAppMetrics(resourcePreempted,
- numNonAMContainerPreempted, numAMContainerPreempted);
+ numNonAMContainerPreempted, numAMContainerPreempted,
+ memorySeconds, vcoreSeconds);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
index 645db16..5091470 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
@@ -24,12 +24,17 @@ public class RMAppMetrics {
final Resource resourcePreempted;
final int numNonAMContainersPreempted;
final int numAMContainersPreempted;
+ final long memorySeconds;
+ final long vcoreSeconds;
public RMAppMetrics(Resource resourcePreempted,
- int numNonAMContainersPreempted, int numAMContainersPreempted) {
+ int numNonAMContainersPreempted, int numAMContainersPreempted,
+ long memorySeconds, long vcoreSeconds) {
this.resourcePreempted = resourcePreempted;
this.numNonAMContainersPreempted = numNonAMContainersPreempted;
this.numAMContainersPreempted = numAMContainersPreempted;
+ this.memorySeconds = memorySeconds;
+ this.vcoreSeconds = vcoreSeconds;
}
public Resource getResourcePreempted() {
@@ -43,4 +48,12 @@ public class RMAppMetrics {
public int getNumAMContainersPreempted() {
return numAMContainersPreempted;
}
+
+ public long getMemorySeconds() {
+ return memorySeconds;
+ }
+
+ public long getVcoreSeconds() {
+ return vcoreSeconds;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
new file mode 100644
index 0000000..f0c2b34
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+@Private
+public class AggregateAppResourceUsage {
+ long memorySeconds;
+ long vcoreSeconds;
+
+ public AggregateAppResourceUsage(long memorySeconds, long vcoreSeconds) {
+ this.memorySeconds = memorySeconds;
+ this.vcoreSeconds = vcoreSeconds;
+ }
+
+ /**
+ * @return the memorySeconds
+ */
+ public long getMemorySeconds() {
+ return memorySeconds;
+ }
+
+ /**
+ * @param memorySeconds the memorySeconds to set
+ */
+ public void setMemorySeconds(long memorySeconds) {
+ this.memorySeconds = memorySeconds;
+ }
+
+ /**
+ * @return the vcoreSeconds
+ */
+ public long getVcoreSeconds() {
+ return vcoreSeconds;
+ }
+
+ /**
+ * @param vcoreSeconds the vcoreSeconds to set
+ */
+ public void setVcoreSeconds(long vcoreSeconds) {
+ this.vcoreSeconds = vcoreSeconds;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 19fc800..93db340 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -85,6 +85,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
@@ -430,7 +431,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.proxiedTrackingUrl = generateProxyUriWithScheme(null);
this.maybeLastAttempt = maybeLastAttempt;
this.stateMachine = stateMachineFactory.make(this);
- this.attemptMetrics = new RMAppAttemptMetrics(applicationAttemptId);
+ this.attemptMetrics =
+ new RMAppAttemptMetrics(applicationAttemptId, rmContext);
}
@Override
@@ -704,6 +706,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
if (report == null) {
report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
}
+ AggregateAppResourceUsage resUsage =
+ this.attemptMetrics.getAggregateAppResourceUsage();
+ report.setMemorySeconds(resUsage.getMemorySeconds());
+ report.setVcoreSeconds(resUsage.getVcoreSeconds());
return report;
} finally {
this.readLock.unlock();
@@ -733,6 +739,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.proxiedTrackingUrl = generateProxyUriWithScheme(originalTrackingUrl);
this.finalStatus = attemptState.getFinalApplicationStatus();
this.startTime = attemptState.getStartTime();
+ this.attemptMetrics.updateAggregateAppResourceUsage(
+ attemptState.getMemorySeconds(),attemptState.getVcoreSeconds());
}
public void transferStateFromPreviousAttempt(RMAppAttempt attempt) {
@@ -1017,12 +1025,14 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
default:
break;
}
-
+ AggregateAppResourceUsage resUsage =
+ this.attemptMetrics.getAggregateAppResourceUsage();
RMStateStore rmStore = rmContext.getStateStore();
ApplicationAttemptState attemptState =
new ApplicationAttemptState(applicationAttemptId, getMasterContainer(),
rmStore.getCredentialsFromAppAttempt(this), startTime,
- stateToBeStored, finalTrackingUrl, diags, finalStatus, exitStatus);
+ stateToBeStored, finalTrackingUrl, diags, finalStatus, exitStatus,
+ resUsage.getMemorySeconds(), resUsage.getVcoreSeconds());
LOG.info("Updating application attempt " + applicationAttemptId
+ " with final state: " + targetedFinalState + ", and exit status: "
+ exitStatus);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 96b41c3..0e60fd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -27,7 +28,9 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -42,12 +45,17 @@ public class RMAppAttemptMetrics {
private ReadLock readLock;
private WriteLock writeLock;
-
- public RMAppAttemptMetrics(ApplicationAttemptId attemptId) {
+ private AtomicLong finishedMemorySeconds = new AtomicLong(0);
+ private AtomicLong finishedVcoreSeconds = new AtomicLong(0);
+ private RMContext rmContext;
+
+ public RMAppAttemptMetrics(ApplicationAttemptId attemptId,
+ RMContext rmContext) {
this.attemptId = attemptId;
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
+ this.rmContext = rmContext;
}
public void updatePreemptionInfo(Resource resource, RMContainer container) {
@@ -94,4 +102,28 @@ public class RMAppAttemptMetrics {
public boolean getIsPreempted() {
return this.isPreempted.get();
}
+
+ public AggregateAppResourceUsage getAggregateAppResourceUsage() {
+ long memorySeconds = finishedMemorySeconds.get();
+ long vcoreSeconds = finishedVcoreSeconds.get();
+
+ // Only add in the running containers if this is the active attempt.
+ RMAppAttempt currentAttempt = rmContext.getRMApps()
+ .get(attemptId.getApplicationId()).getCurrentAppAttempt();
+ if (currentAttempt.getAppAttemptId().equals(attemptId)) {
+ ApplicationResourceUsageReport appResUsageReport = rmContext
+ .getScheduler().getAppResourceUsageReport(attemptId);
+ if (appResUsageReport != null) {
+ memorySeconds += appResUsageReport.getMemorySeconds();
+ vcoreSeconds += appResUsageReport.getVcoreSeconds();
+ }
+ }
+ return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds);
+ }
+
+ public void updateAggregateAppResourceUsage(long finishedMemorySeconds,
+ long finishedVcoreSeconds) {
+ this.finishedMemorySeconds.addAndGet(finishedMemorySeconds);
+ this.finishedVcoreSeconds.addAndGet(finishedVcoreSeconds);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index eef361f..e7bb98e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -24,6 +24,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent;
@@ -488,7 +490,7 @@ public class RMContainerImpl implements RMContainer {
// Inform AppAttempt
// container.getContainer() can return null when a RMContainer is a
// reserved container
- updateMetricsIfPreempted(container);
+ updateAttemptMetrics(container);
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
container.appAttemptId, finishedEvent.getRemoteContainerStatus()));
@@ -497,19 +499,27 @@ public class RMContainerImpl implements RMContainer {
container);
}
- private static void updateMetricsIfPreempted(RMContainerImpl container) {
+ private static void updateAttemptMetrics(RMContainerImpl container) {
// If this is a preempted container, update preemption metrics
+ Resource resource = container.getContainer().getResource();
+ RMAppAttempt rmAttempt = container.rmContext.getRMApps()
+ .get(container.getApplicationAttemptId().getApplicationId())
+ .getCurrentAppAttempt();
if (ContainerExitStatus.PREEMPTED == container.finishedStatus
.getExitStatus()) {
-
- Resource resource = container.getContainer().getResource();
- RMAppAttempt rmAttempt =
- container.rmContext.getRMApps()
- .get(container.getApplicationAttemptId().getApplicationId())
- .getCurrentAppAttempt();
rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
container);
}
+
+ if (rmAttempt != null) {
+ long usedMillis = container.finishTime - container.creationTime;
+ long memorySeconds = resource.getMemory()
+ * usedMillis / DateUtils.MILLIS_PER_SECOND;
+ long vcoreSeconds = resource.getVirtualCores()
+ * usedMillis / DateUtils.MILLIS_PER_SECOND;
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 933f456..7032e3c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -43,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
@@ -69,6 +71,11 @@ public class SchedulerApplicationAttempt {
private static final Log LOG = LogFactory
.getLog(SchedulerApplicationAttempt.class);
+ private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000;
+ protected long lastMemoryAggregateAllocationUpdateTime = 0;
+ private long lastMemorySeconds = 0;
+ private long lastVcoreSeconds = 0;
+
protected final AppSchedulingInfo appSchedulingInfo;
protected Map<ContainerId, RMContainer> liveContainers =
@@ -505,12 +512,38 @@ public class SchedulerApplicationAttempt {
lastScheduledContainer.put(priority, currentTimeMs);
schedulingOpportunities.setCount(priority, 0);
}
-
+
+ synchronized AggregateAppResourceUsage getRunningAggregateAppResourceUsage() {
+ long currentTimeMillis = System.currentTimeMillis();
+ // Don't walk the whole container list if the resources were computed
+ // recently.
+ if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime)
+ > MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) {
+ long memorySeconds = 0;
+ long vcoreSeconds = 0;
+ for (RMContainer rmContainer : this.liveContainers.values()) {
+ long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
+ Resource resource = rmContainer.getContainer().getResource();
+ memorySeconds += resource.getMemory() * usedMillis /
+ DateUtils.MILLIS_PER_SECOND;
+ vcoreSeconds += resource.getVirtualCores() * usedMillis
+ / DateUtils.MILLIS_PER_SECOND;
+ }
+
+ lastMemoryAggregateAllocationUpdateTime = currentTimeMillis;
+ lastMemorySeconds = memorySeconds;
+ lastVcoreSeconds = vcoreSeconds;
+ }
+ return new AggregateAppResourceUsage(lastMemorySeconds, lastVcoreSeconds);
+ }
+
public synchronized ApplicationResourceUsageReport getResourceUsageReport() {
+ AggregateAppResourceUsage resUsage = getRunningAggregateAppResourceUsage();
return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
- reservedContainers.size(), Resources.clone(currentConsumption),
- Resources.clone(currentReservation),
- Resources.add(currentConsumption, currentReservation));
+ reservedContainers.size(), Resources.clone(currentConsumption),
+ Resources.clone(currentReservation),
+ Resources.add(currentConsumption, currentReservation),
+ resUsage.getMemorySeconds(), resUsage.getVcoreSeconds());
}
public synchronized Map<ContainerId, RMContainer> getLiveContainersMap() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 846d1e1..167dcd8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -106,6 +106,9 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
queue.getMetrics().releaseResources(getUser(), 1, containerResource);
Resources.subtractFrom(currentConsumption, containerResource);
+ // Clear resource utilization metrics cache.
+ lastMemoryAggregateAllocationUpdateTime = -1;
+
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index bf54376..825c398 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -146,6 +146,9 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// remove from preemption map if it is completed
preemptionMap.remove(rmContainer);
+
+ // Clear resource utilization metrics cache.
+ lastMemoryAggregateAllocationUpdateTime = -1;
}
private synchronized void unreserveInternal(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
index b9f2dda..c427ccf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
@@ -146,7 +146,10 @@ public class AppBlock extends HtmlBlock {
attemptMetrics.getResourcePreempted())
._("Number of Non-AM Containers Preempted from Current Attempt:",
String.valueOf(attemptMetrics
- .getNumNonAMContainersPreempted()));
+ .getNumNonAMContainersPreempted()))
+ ._("Aggregate Resource Allocation:",
+ String.format("%d MB-seconds, %d vcore-seconds",
+ appMerics.getMemorySeconds(), appMerics.getVcoreSeconds()));
pdiv._();
Collection<RMAppAttempt> attempts = rmApp.getAppAttempts().values();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index c7354ce..2b0dedc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -79,6 +79,8 @@ public class AppInfo {
protected int allocatedMB;
protected int allocatedVCores;
protected int runningContainers;
+ protected long memorySeconds;
+ protected long vcoreSeconds;
// preemption info fields
protected int preemptedResourceMB;
@@ -165,6 +167,8 @@ public class AppInfo {
appMetrics.getNumNonAMContainersPreempted();
preemptedResourceVCores =
appMetrics.getResourcePreempted().getVirtualCores();
+ memorySeconds = appMetrics.getMemorySeconds();
+ vcoreSeconds = appMetrics.getVcoreSeconds();
}
}
@@ -287,4 +291,12 @@ public class AppInfo {
public int getNumAMContainersPreempted() {
return numAMContainerPreempted;
}
+
+ public long getMemorySeconds() {
+ return memorySeconds;
+ }
+
+ public long getVcoreSeconds() {
+ return vcoreSeconds;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
index eab6af1..5125a27 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
@@ -78,6 +78,8 @@ message ApplicationAttemptStateDataProto {
optional int64 start_time = 7;
optional FinalApplicationStatusProto final_application_status = 8;
optional int32 am_container_exit_status = 9 [default = -1000];
+ optional int64 memory_seconds = 10;
+ optional int64 vcore_seconds = 11;
}
message EpochProto {
@@ -87,4 +89,4 @@ message EpochProto {
message AMRMTokenSecretManagerStateProto {
optional MasterKeyProto current_master_key = 1;
optional MasterKeyProto next_master_key = 2;
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83be3ad4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 4f4da37..571c96f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRes
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
@@ -79,6 +80,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -223,7 +225,7 @@ public class TestClientRMService {
}
@Test
- public void testGetApplicationReport() throws YarnException {
+ public void testNonExistingApplicationReport() throws YarnException {
RMContext rmContext = mock(RMContext.class);
when(rmContext.getRMApps()).thenReturn(
new ConcurrentHashMap<ApplicationId, RMApp>());
@@ -242,6 +244,38 @@ public class TestClientRMService {
+ "' doesn't exist in RM.");
}
}
+
+ @Test
+ public void testGetApplicationReport() throws Exception {
+ YarnScheduler yarnScheduler = mock(YarnScheduler.class);
+ RMContext rmContext = mock(RMContext.class);
+ mockRMContext(yarnScheduler, rmContext);
+
+ ApplicationId appId1 = getApplicationId(1);
+
+ ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
+ when(
+ mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
+ ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);
+
+ ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
+ null, mockAclsManager, null, null);
+ try {
+ RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+ GetApplicationReportRequest request = recordFactory
+ .newRecordInstance(GetApplicationReportRequest.class);
+ request.setApplicationId(appId1);
+ GetApplicationReportResponse response =
+ rmService.getApplicationReport(request);
+ ApplicationReport report = response.getApplicationReport();
+ ApplicationResourceUsageReport usageReport =
+ report.getApplicationResourceUsageReport();
+ Assert.assertEquals(10, usageReport.getMemorySeconds());
+ Assert.assertEquals(3, usageReport.getVcoreSeconds());
+ } finally {
+ rmService.close();
+ }
+ }
@Test
public void testGetApplicationAttemptReport() throws YarnException,
@@ -1065,11 +1099,11 @@ public class TestClientRMService {
ApplicationId applicationId3 = getApplicationId(3);
YarnConfiguration config = new YarnConfiguration();
apps.put(applicationId1, getRMApp(rmContext, yarnScheduler, applicationId1,
- config, "testqueue"));
+ config, "testqueue", 10, 3));
apps.put(applicationId2, getRMApp(rmContext, yarnScheduler, applicationId2,
- config, "a"));
+ config, "a", 20, 2));
apps.put(applicationId3, getRMApp(rmContext, yarnScheduler, applicationId3,
- config, "testqueue"));
+ config, "testqueue", 40, 5));
return apps;
}
@@ -1091,12 +1125,26 @@ public class TestClientRMService {
}
private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler,
- ApplicationId applicationId3, YarnConfiguration config, String queueName) {
+ ApplicationId applicationId3, YarnConfiguration config, String queueName,
+ final long memorySeconds, final long vcoreSeconds) {
ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class);
when(asContext.getMaxAppAttempts()).thenReturn(1);
RMAppImpl app = spy(new RMAppImpl(applicationId3, rmContext, config, null,
null, queueName, asContext, yarnScheduler, null,
- System.currentTimeMillis(), "YARN", null));
+ System.currentTimeMillis(), "YARN", null) {
+ @Override
+ public ApplicationReport createAndGetApplicationReport(
+ String clientUserName, boolean allowAccess) {
+ ApplicationReport report = super.createAndGetApplicationReport(
+ clientUserName, allowAccess);
+ ApplicationResourceUsageReport usageReport =
+ report.getApplicationResourceUsageReport();
+ usageReport.setMemorySeconds(memorySeconds);
+ usageReport.setVcoreSeconds(vcoreSeconds);
+ report.setApplicationResourceUsageReport(usageReport);
+ return report;
+ }
+ });
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
ApplicationId.newInstance(123456, 1), 1);
RMAppAttemptImpl rmAppAttemptImpl = spy(new RMAppAttemptImpl(attemptId,