You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2016/09/07 04:45:23 UTC
[01/22] hadoop git commit: YARN-5608. TestAMRMClient.setup() fails
with ArrayOutOfBoundsException. Contributed by Daniel Templeton. [Forced
Update!]
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-12756 8b68b11bd -> e671a0f52 (forced update)
YARN-5608. TestAMRMClient.setup() fails with ArrayOutOfBoundsException. Contributed by Daniel Templeton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9557127
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9557127
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9557127
Branch: refs/heads/HADOOP-12756
Commit: f9557127b3660e138cab8d9535a589677809384a
Parents: 07650bc
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Sep 6 14:09:45 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Sep 6 14:09:45 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/api/impl/TestAMRMClient.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9557127/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 38178a4..239189a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -145,8 +145,13 @@ public class TestAMRMClient {
yarnClient.start();
// get node info
+ assertTrue("All node managers did not connect to the RM within the "
+ + "allotted 5-second timeout",
+ yarnCluster.waitForNodeManagersToConnect(5000L));
nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
-
+ assertEquals("Not all node managers were reported running",
+ nodeCount, nodeReports.size());
+
priority = Priority.newInstance(1);
priority2 = Priority.newInstance(2);
capability = Resource.newInstance(1024, 1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/22] hadoop git commit: HADOOP-12756. Incorporate Aliyun OSS file
system implementation. Contributed by Mingfei Shi and Lin Zhou
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java
new file mode 100644
index 0000000..3951529
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+
+/**
+ * Tests regular and multi-part upload functionality for AliyunOSSOutputStream.
+ */
+public class TestOSSOutputStream {
+ private FileSystem fs;
+ private static String testRootPath = OSSTestUtils.generateUniqueTestPath();
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setLong(Constants.MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, 5 * 1024 * 1024);
+ conf.setInt(Constants.MULTIPART_UPLOAD_SIZE_KEY, 5 * 1024 * 1024);
+ fs = OSSTestUtils.createTestFileSystem(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(new Path(testRootPath), true);
+ }
+ }
+
+ protected Path getTestPath() {
+ return new Path(testRootPath + "/testoss");
+ }
+
+ @Test
+ public void testRegularUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
+ }
+
+ @Test
+ public void testMultiPartUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 * 1024);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java
new file mode 100644
index 0000000..8214b9f
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.aliyun.oss.OSSTestUtils;
+import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
+
+/**
+ * The contract of OSS: only enabled if the test bucket is provided.
+ */
+public class OSSContract extends AbstractBondedFSContract {
+
+ public static final String CONTRACT_XML = "contract/oss.xml";
+ public static final String CONTRACT_TEST_OSS_FS_NAME =
+ "fs.contract.test.fs.oss";
+
+ private static String testPath = OSSTestUtils.generateUniqueTestPath();
+
+ public OSSContract(Configuration conf) {
+ super(conf);
+ //insert the base features
+ addConfResource(CONTRACT_XML);
+ }
+
+ @Override
+ public String getScheme() {
+ return "oss";
+ }
+
+ @Override
+ public Path getTestPath() {
+ Path path = new Path(testPath);
+ return path;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java
new file mode 100644
index 0000000..cc5a2d1
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+/**
+ * OSS contract creating tests.
+ */
+public class TestOSSContractCreate extends AbstractContractCreateTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OSSContract(conf);
+ }
+
+ @Override
+ public void testOverwriteEmptyDirectory() throws Throwable {
+ ContractTestUtils.skip(
+ "blobstores can't distinguish empty directories from files");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java
new file mode 100644
index 0000000..6a1eb40
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * OSS contract deleting tests.
+ */
+public class TestOSSContractDelete extends AbstractContractDeleteTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java
new file mode 100644
index 0000000..1dcb7f0
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * OSS contract directory tests.
+ */
+public class TestOSSContractMkdir extends AbstractContractMkdirTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java
new file mode 100644
index 0000000..ee0c055
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * OSS contract opening file tests.
+ */
+public class TestOSSContractOpen extends AbstractContractOpenTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java
new file mode 100644
index 0000000..634fcf1
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * OSS contract renaming tests.
+ */
+public class TestOSSContractRename extends AbstractContractRenameTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OSSContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java
new file mode 100644
index 0000000..40ea772
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * OSS contract seeking tests.
+ */
+public class TestOSSContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml
new file mode 100644
index 0000000..2bc34b7
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml
@@ -0,0 +1,105 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>fs.contract.test.random-seek-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-blobstore</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-case-sensitive</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-returns-false-if-source-missing</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-remove-dest-if-empty-dir</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-append</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-directory-delete</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-rename</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-block-locality</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-concat</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek-on-closed-file</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rejects-seek-past-eof</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-strict-exceptions</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-unix-permissions</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-overwrites-dest</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.oss.multipart.download.size</name>
+ <value>102400</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml
new file mode 100644
index 0000000..fa4118c
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+
+ <property>
+ <name>hadoop.tmp.dir</name>
+ <value>target/build/test</value>
+ <description>A base for other temporary directories.</description>
+ <final>true</final>
+ </property>
+
+ <!-- Turn security off for tests by default -->
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
+ </property>
+
+ <!--
+ To run these tests.
+
+ # Create a file auth-keys.xml - DO NOT ADD TO REVISION CONTROL
+ # add the property test.fs.oss.name to point to an OSS filesystem URL
+ # Add the credentials for the service you are testing against
+ -->
+ <include xmlns="http://www.w3.org/2001/XInclude" href="auth-keys.xml">
+ <fallback/>
+ </include>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
new file mode 100644
index 0000000..bb5cbe5
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 899a945..14fa9f0 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -102,6 +102,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aliyun</artifactId>
+ <scope>compile</scope>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-sls</artifactId>
<scope>compile</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index db002f4..e7e876b 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -47,6 +47,7 @@
<module>hadoop-aws</module>
<module>hadoop-kafka</module>
<module>hadoop-azure-datalake</module>
+ <module>hadoop-aliyun</module>
</modules>
<build>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/22] hadoop git commit: HDFS-9038. DFS reserved space is
erroneously counted towards non-DFS used. (Brahma Reddy Battula)
Posted by sh...@apache.org.
HDFS-9038. DFS reserved space is erroneously counted towards non-DFS used. (Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f23abfa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f23abfa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f23abfa
Branch: refs/heads/HADOOP-12756
Commit: 5f23abfa30ea29a5474513c463b4d462c0e824ee
Parents: f6c0b75
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Sep 6 13:37:21 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Sep 6 13:37:21 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/DatanodeInfo.java | 33 +++++++++++++---
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 41 +++++++++++++-------
.../hdfs/server/protocol/StorageReport.java | 10 ++++-
.../src/main/proto/hdfs.proto | 2 +
.../blockmanagement/DatanodeDescriptor.java | 3 ++
.../server/blockmanagement/DatanodeStats.java | 6 ++-
.../blockmanagement/DatanodeStorageInfo.java | 8 +++-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 3 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 37 ++++++++++++++++--
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 24 ++++++++++++
.../blockmanagement/BlockManagerTestUtil.java | 2 +-
.../blockmanagement/TestBlockManager.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 2 +-
.../extdataset/ExternalDatasetImpl.java | 2 +-
.../fsdataset/impl/TestFsVolumeList.java | 40 +++++++++++++++++++
.../server/namenode/NNThroughputBenchmark.java | 4 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 2 +-
.../namenode/TestNamenodeCapacityReport.java | 36 ++++++++++++++---
.../namenode/metrics/TestNameNodeMetrics.java | 4 +-
19 files changed, 219 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index cd32a53..e9ee8b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -43,6 +43,7 @@ import static org.apache.hadoop.hdfs.DFSUtilClient.percent2String;
public class DatanodeInfo extends DatanodeID implements Node {
private long capacity;
private long dfsUsed;
+ private long nonDfsUsed;
private long remaining;
private long blockPoolUsed;
private long cacheCapacity;
@@ -89,6 +90,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
super(from);
this.capacity = from.getCapacity();
this.dfsUsed = from.getDfsUsed();
+ this.nonDfsUsed = from.getNonDfsUsed();
this.remaining = from.getRemaining();
this.blockPoolUsed = from.getBlockPoolUsed();
this.cacheCapacity = from.getCacheCapacity();
@@ -105,6 +107,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
super(nodeID);
this.capacity = 0L;
this.dfsUsed = 0L;
+ this.nonDfsUsed = 0L;
this.remaining = 0L;
this.blockPoolUsed = 0L;
this.cacheCapacity = 0L;
@@ -158,10 +161,26 @@ public class DatanodeInfo extends DatanodeID implements Node {
final int xceiverCount, final String networkLocation,
final AdminStates adminState,
final String upgradeDomain) {
- super(ipAddr, hostName, datanodeUuid, xferPort, infoPort,
- infoSecurePort, ipcPort);
+ this(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
+ ipcPort, capacity, dfsUsed, 0L, remaining, blockPoolUsed,
+ cacheCapacity, cacheUsed, lastUpdate, lastUpdateMonotonic,
+ xceiverCount, networkLocation, adminState, upgradeDomain);
+ }
+
+ /** Constructor. */
+ public DatanodeInfo(final String ipAddr, final String hostName,
+ final String datanodeUuid, final int xferPort, final int infoPort,
+ final int infoSecurePort, final int ipcPort, final long capacity,
+ final long dfsUsed, final long nonDfsUsed, final long remaining,
+ final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
+ final long lastUpdate, final long lastUpdateMonotonic,
+ final int xceiverCount, final String networkLocation,
+ final AdminStates adminState, final String upgradeDomain) {
+ super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
+ ipcPort);
this.capacity = capacity;
this.dfsUsed = dfsUsed;
+ this.nonDfsUsed = nonDfsUsed;
this.remaining = remaining;
this.blockPoolUsed = blockPoolUsed;
this.cacheCapacity = cacheCapacity;
@@ -174,7 +193,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.upgradeDomain = upgradeDomain;
}
- /** Network location name */
+ /** Network location name. */
@Override
public String getName() {
return getXferAddr();
@@ -191,8 +210,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
/** The used space by the data node. */
public long getNonDfsUsed() {
- long nonDFSUsed = capacity - dfsUsed - remaining;
- return nonDFSUsed < 0 ? 0 : nonDFSUsed;
+ return nonDfsUsed;
}
/** The used space by the data node as percentage of present capacity */
@@ -282,6 +300,11 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.dfsUsed = dfsUsed;
}
+ /** Sets the nondfs-used space for the datanode. */
+ public void setNonDfsUsed(long nonDfsUsed) {
+ this.nonDfsUsed = nonDfsUsed;
+ }
+
/** Sets raw free space. */
public void setRemaining(long remaining) {
this.remaining = remaining;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 7df68e7..246b5a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -289,6 +289,7 @@ public class PBHelperClient {
.setId(convert((DatanodeID) info))
.setCapacity(info.getCapacity())
.setDfsUsed(info.getDfsUsed())
+ .setNonDfsUsed(info.getNonDfsUsed())
.setRemaining(info.getRemaining())
.setBlockPoolUsed(info.getBlockPoolUsed())
.setCacheCapacity(info.getCacheCapacity())
@@ -581,15 +582,24 @@ public class PBHelperClient {
}
static public DatanodeInfo convert(DatanodeInfoProto di) {
- if (di == null) return null;
- return new DatanodeInfo(
- convert(di.getId()),
- di.hasLocation() ? di.getLocation() : null,
- di.getCapacity(), di.getDfsUsed(), di.getRemaining(),
- di.getBlockPoolUsed(), di.getCacheCapacity(), di.getCacheUsed(),
- di.getLastUpdate(), di.getLastUpdateMonotonic(),
- di.getXceiverCount(), convert(di.getAdminState()),
+ if (di == null) {
+ return null;
+ }
+ DatanodeInfo dinfo = new DatanodeInfo(convert(di.getId()),
+ di.hasLocation() ? di.getLocation() : null, di.getCapacity(),
+ di.getDfsUsed(), di.getRemaining(), di.getBlockPoolUsed(),
+ di.getCacheCapacity(), di.getCacheUsed(), di.getLastUpdate(),
+ di.getLastUpdateMonotonic(), di.getXceiverCount(),
+ convert(di.getAdminState()),
di.hasUpgradeDomain() ? di.getUpgradeDomain() : null);
+ if (di.hasNonDfsUsed()) {
+ dinfo.setNonDfsUsed(di.getNonDfsUsed());
+ } else {
+ // use the legacy way for older datanodes
+ long nonDFSUsed = di.getCapacity() - di.getDfsUsed() - di.getRemaining();
+ dinfo.setNonDfsUsed(nonDFSUsed < 0 ? 0 : nonDFSUsed);
+ }
+ return dinfo;
}
public static StorageType[] convertStorageTypes(
@@ -1556,12 +1566,12 @@ public class PBHelperClient {
}
public static StorageReport convert(StorageReportProto p) {
- return new StorageReport(
- p.hasStorage() ?
- convert(p.getStorage()) :
- new DatanodeStorage(p.getStorageUuid()),
- p.getFailed(), p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
- p.getBlockPoolUsed());
+ long nonDfsUsed = p.hasNonDfsUsed() ? p.getNonDfsUsed() : p.getCapacity()
+ - p.getDfsUsed() - p.getRemaining();
+ return new StorageReport(p.hasStorage() ? convert(p.getStorage())
+ : new DatanodeStorage(p.getStorageUuid()), p.getFailed(),
+ p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
+ p.getBlockPoolUsed(), nonDfsUsed);
}
public static DatanodeStorage convert(DatanodeStorageProto s) {
@@ -2136,7 +2146,8 @@ public class PBHelperClient {
.setBlockPoolUsed(r.getBlockPoolUsed()).setCapacity(r.getCapacity())
.setDfsUsed(r.getDfsUsed()).setRemaining(r.getRemaining())
.setStorageUuid(r.getStorage().getStorageID())
- .setStorage(convert(r.getStorage()));
+ .setStorage(convert(r.getStorage()))
+ .setNonDfsUsed(r.getNonDfsUsed());
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
index 3fbc424..7042b58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
@@ -25,17 +25,19 @@ public class StorageReport {
private final boolean failed;
private final long capacity;
private final long dfsUsed;
+ private final long nonDfsUsed;
private final long remaining;
private final long blockPoolUsed;
public static final StorageReport[] EMPTY_ARRAY = {};
- public StorageReport(DatanodeStorage storage, boolean failed,
- long capacity, long dfsUsed, long remaining, long bpUsed) {
+ public StorageReport(DatanodeStorage storage, boolean failed, long capacity,
+ long dfsUsed, long remaining, long bpUsed, long nonDfsUsed) {
this.storage = storage;
this.failed = failed;
this.capacity = capacity;
this.dfsUsed = dfsUsed;
+ this.nonDfsUsed = nonDfsUsed;
this.remaining = remaining;
this.blockPoolUsed = bpUsed;
}
@@ -56,6 +58,10 @@ public class StorageReport {
return dfsUsed;
}
+ public long getNonDfsUsed() {
+ return nonDfsUsed;
+ }
+
public long getRemaining() {
return remaining;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 7de8550..1141fcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -101,6 +101,7 @@ message DatanodeInfoProto {
optional uint64 cacheUsed = 12 [default = 0];
optional uint64 lastUpdateMonotonic = 13 [default = 0];
optional string upgradeDomain = 14;
+ optional uint64 nonDfsUsed = 15;
}
/**
@@ -125,6 +126,7 @@ message StorageReportProto {
optional uint64 remaining = 5 [ default = 0 ];
optional uint64 blockPoolUsed = 6 [ default = 0 ];
optional DatanodeStorageProto storage = 7; // supersedes StorageUuid
+ optional uint64 nonDfsUsed = 8;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 1646129..c74d7c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -417,6 +417,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
long totalRemaining = 0;
long totalBlockPoolUsed = 0;
long totalDfsUsed = 0;
+ long totalNonDfsUsed = 0;
Set<DatanodeStorageInfo> failedStorageInfos = null;
// Decide if we should check for any missing StorageReport and mark it as
@@ -477,6 +478,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
totalRemaining += report.getRemaining();
totalBlockPoolUsed += report.getBlockPoolUsed();
totalDfsUsed += report.getDfsUsed();
+ totalNonDfsUsed += report.getNonDfsUsed();
}
rollBlocksScheduled(getLastUpdateMonotonic());
@@ -485,6 +487,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
setRemaining(totalRemaining);
setBlockPoolUsed(totalBlockPoolUsed);
setDfsUsed(totalDfsUsed);
+ setNonDfsUsed(totalNonDfsUsed);
if (checkFailedStorages) {
updateFailedStorage(failedStorageInfos);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
index 0d4e235..8386b27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
@@ -35,6 +35,7 @@ class DatanodeStats {
private final StorageTypeStatsMap statsMap = new StorageTypeStatsMap();
private long capacityTotal = 0L;
private long capacityUsed = 0L;
+ private long capacityUsedNonDfs = 0L;
private long capacityRemaining = 0L;
private long blockPoolUsed = 0L;
private int xceiverCount = 0;
@@ -49,6 +50,7 @@ class DatanodeStats {
xceiverCount += node.getXceiverCount();
if (node.isInService()) {
capacityUsed += node.getDfsUsed();
+ capacityUsedNonDfs += node.getNonDfsUsed();
blockPoolUsed += node.getBlockPoolUsed();
nodesInService++;
nodesInServiceXceiverCount += node.getXceiverCount();
@@ -77,6 +79,7 @@ class DatanodeStats {
xceiverCount -= node.getXceiverCount();
if (node.isInService()) {
capacityUsed -= node.getDfsUsed();
+ capacityUsedNonDfs -= node.getNonDfsUsed();
blockPoolUsed -= node.getBlockPoolUsed();
nodesInService--;
nodesInServiceXceiverCount -= node.getXceiverCount();
@@ -159,8 +162,7 @@ class DatanodeStats {
}
synchronized long getCapacityUsedNonDFS() {
- final long nonDFSUsed = capacityTotal - capacityRemaining - capacityUsed;
- return nonDFSUsed < 0L? 0L : nonDFSUsed;
+ return capacityUsedNonDfs;
}
synchronized float getCapacityUsedPercent() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 843a8d5..d98a2c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -93,6 +93,7 @@ public class DatanodeStorageInfo {
private long capacity;
private long dfsUsed;
+ private long nonDfsUsed;
private volatile long remaining;
private long blockPoolUsed;
@@ -202,6 +203,10 @@ public class DatanodeStorageInfo {
return dfsUsed;
}
+ long getNonDfsUsed() {
+ return nonDfsUsed;
+ }
+
long getRemaining() {
return remaining;
}
@@ -283,6 +288,7 @@ public class DatanodeStorageInfo {
void updateState(StorageReport r) {
capacity = r.getCapacity();
dfsUsed = r.getDfsUsed();
+ nonDfsUsed = r.getNonDfsUsed();
remaining = r.getRemaining();
blockPoolUsed = r.getBlockPoolUsed();
}
@@ -332,7 +338,7 @@ public class DatanodeStorageInfo {
StorageReport toStorageReport() {
return new StorageReport(
new DatanodeStorage(storageID, state, storageType),
- false, capacity, dfsUsed, remaining, blockPoolUsed);
+ false, capacity, dfsUsed, remaining, blockPoolUsed, nonDfsUsed);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index e0d2baf..ffd2f8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -164,7 +164,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
volume.getCapacity(),
volume.getDfsUsed(),
volume.getAvailable(),
- volume.getBlockPoolUsed(bpid));
+ volume.getBlockPoolUsed(bpid),
+ volume.getNonDfsUsed());
reports.add(sr);
} catch (ClosedChannelException e) {
continue;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 59c9ed4..afcc5dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -385,15 +385,46 @@ public class FsVolumeImpl implements FsVolumeSpi {
*/
@Override
public long getAvailable() throws IOException {
- long remaining = getCapacity() - getDfsUsed() - reservedForReplicas.get();
- long available = usage.getAvailable() - reserved
- - reservedForReplicas.get();
+ long remaining = getCapacity() - getDfsUsed() - getReservedForReplicas();
+ long available = usage.getAvailable() - getRemainingReserved()
+ - getReservedForReplicas();
if (remaining > available) {
remaining = available;
}
return (remaining > 0) ? remaining : 0;
}
+ long getActualNonDfsUsed() throws IOException {
+ return usage.getUsed() - getDfsUsed();
+ }
+
+ private long getRemainingReserved() throws IOException {
+ long actualNonDfsUsed = getActualNonDfsUsed();
+ if (actualNonDfsUsed < reserved) {
+ return reserved - actualNonDfsUsed;
+ }
+ return 0L;
+ }
+
+ /**
+ * Unplanned Non-DFS usage, i.e. Extra usage beyond reserved.
+ *
+ * @return
+ * @throws IOException
+ */
+ public long getNonDfsUsed() throws IOException {
+ long actualNonDfsUsed = getActualNonDfsUsed();
+ if (actualNonDfsUsed < reserved) {
+ return 0L;
+ }
+ return actualNonDfsUsed - reserved;
+ }
+
+ @VisibleForTesting
+ long getDfAvailable() {
+ return usage.getAvailable();
+ }
+
@VisibleForTesting
public long getReservedForReplicas() {
return reservedForReplicas.get();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index 45cd4ea..af17756 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -720,6 +720,28 @@ public class TestPBHelper {
assertBlockECRecoveryInfoEquals(blkECRecoveryInfo1, iterator.next());
}
+ @Test
+ public void testDataNodeInfoPBHelper() {
+ DatanodeID id = DFSTestUtil.getLocalDatanodeID();
+ DatanodeInfo dnInfos0 = new DatanodeInfo(id);
+ dnInfos0.setCapacity(3500L);
+ dnInfos0.setDfsUsed(1000L);
+ dnInfos0.setNonDfsUsed(2000L);
+ dnInfos0.setRemaining(500L);
+ HdfsProtos.DatanodeInfoProto dnproto = PBHelperClient.convert(dnInfos0);
+ DatanodeInfo dnInfos1 = PBHelperClient.convert(dnproto);
+ compare(dnInfos0, dnInfos1);
+ assertEquals(dnInfos0.getNonDfsUsed(), dnInfos1.getNonDfsUsed());
+
+ //Testing without nonDfs field
+ HdfsProtos.DatanodeInfoProto.Builder b =
+ HdfsProtos.DatanodeInfoProto.newBuilder();
+ b.setId(PBHelperClient.convert(id)).setCapacity(3500L).setDfsUsed(1000L)
+ .setRemaining(500L);
+ DatanodeInfo dnInfos3 = PBHelperClient.convert(b.build());
+ assertEquals(dnInfos0.getNonDfsUsed(), dnInfos3.getNonDfsUsed());
+ }
+
private void assertBlockECRecoveryInfoEquals(
BlockECReconstructionInfo blkECRecoveryInfo1,
BlockECReconstructionInfo blkECRecoveryInfo2) {
@@ -768,4 +790,6 @@ public class TestPBHelper {
compare(dnInfos1[i], dnInfos2[i]);
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
index 1b565ee..5fb3b25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
@@ -294,7 +294,7 @@ public class BlockManagerTestUtil {
StorageReport report = new StorageReport(
dns ,false, storage.getCapacity(),
storage.getDfsUsed(), storage.getRemaining(),
- storage.getBlockPoolUsed());
+ storage.getBlockPoolUsed(), 0);
reports.add(report);
}
return reports.toArray(StorageReport.EMPTY_ARRAY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 1f58f99..942569a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -1200,7 +1200,7 @@ public class TestBlockManager {
StorageReport report = new StorageReport(
dns, true, storageInfo.getCapacity(),
storageInfo.getDfsUsed(), storageInfo.getRemaining(),
- storageInfo.getBlockPoolUsed());
+ storageInfo.getBlockPoolUsed(), 0L);
reports.add(report);
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index c0c76ad..a082fbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -441,7 +441,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
synchronized StorageReport getStorageReport(String bpid) {
return new StorageReport(dnStorage,
false, getCapacity(), getUsed(), getFree(),
- map.get(bpid).getUsed());
+ map.get(bpid).getUsed(), 0L);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
index b8564bb..fcd960a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
@@ -72,7 +72,7 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> {
@Override
public StorageReport[] getStorageReports(String bpid) throws IOException {
StorageReport[] result = new StorageReport[1];
- result[0] = new StorageReport(storage, false, 0, 0, 0, 0);
+ result[0] = new StorageReport(storage, false, 0, 0, 0, 0, 0);
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
index 796d249..3d4c38c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
import java.io.File;
import java.io.IOException;
@@ -177,4 +178,43 @@ public class TestFsVolumeList {
conf, StorageType.DEFAULT);
assertEquals("", 100L, volume4.getReserved());
}
+
+ @Test
+ public void testNonDfsUsedMetricForVolume() throws Exception {
+ File volDir = new File(baseDir, "volume-0");
+ volDir.mkdirs();
+ /*
+ * Lets have the example.
+ * Capacity - 1000
+ * Reserved - 100
+ * DfsUsed - 200
+ * Actual Non-DfsUsed - 300 -->(expected)
+ * ReservedForReplicas - 50
+ */
+ long diskCapacity = 1000L;
+ long duReserved = 100L;
+ long dfsUsage = 200L;
+ long actualNonDfsUsage = 300L;
+ long reservedForReplicas = 50L;
+ conf.setLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY, duReserved);
+ FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", volDir, conf,
+ StorageType.DEFAULT);
+ FsVolumeImpl spyVolume = Mockito.spy(volume);
+ // Set Capacity for testing
+ long testCapacity = diskCapacity - duReserved;
+ spyVolume.setCapacityForTesting(testCapacity);
+ // Mock volume.getDfAvailable()
+ long dfAvailable = diskCapacity - dfsUsage - actualNonDfsUsage;
+ Mockito.doReturn(dfAvailable).when(spyVolume).getDfAvailable();
+ // Mock dfsUsage
+ Mockito.doReturn(dfsUsage).when(spyVolume).getDfsUsed();
+ // Mock reservedForReplcas
+ Mockito.doReturn(reservedForReplicas).when(spyVolume)
+ .getReservedForReplicas();
+ Mockito.doReturn(actualNonDfsUsage).when(spyVolume)
+ .getActualNonDfsUsed();
+ long expectedNonDfsUsage =
+ actualNonDfsUsage - duReserved;
+ assertEquals(expectedNonDfsUsage, spyVolume.getNonDfsUsed());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index be2a678..ad336ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -949,7 +949,7 @@ public class NNThroughputBenchmark implements Tool {
// register datanode
// TODO:FEDERATION currently a single block pool is supported
StorageReport[] rep = { new StorageReport(storage, false,
- DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
+ DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0L) };
DatanodeCommand[] cmds = dataNodeProto.sendHeartbeat(dnRegistration, rep,
0L, 0L, 0, 0, 0, null, true).getCommands();
if(cmds != null) {
@@ -998,7 +998,7 @@ public class NNThroughputBenchmark implements Tool {
int replicateBlocks() throws IOException {
// register datanode
StorageReport[] rep = { new StorageReport(storage,
- false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
+ false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0) };
DatanodeCommand[] cmds = dataNodeProto.sendHeartbeat(dnRegistration,
rep, 0L, 0L, 0, 0, 0, null, true).getCommands();
if (cmds != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
index 341a149..14b14b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
@@ -128,7 +128,7 @@ public class TestDeadDatanode {
// that asks datanode to register again
StorageReport[] rep = { new StorageReport(
new DatanodeStorage(reg.getDatanodeUuid()),
- false, 0, 0, 0, 0) };
+ false, 0, 0, 0, 0, 0) };
DatanodeCommand[] cmd =
dnp.sendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null, true).getCommands();
assertEquals(1, cmd.length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
index 99323f5..38ec9f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
@@ -71,7 +71,6 @@ public class TestNamenodeCapacityReport {
try {
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
-
final FSNamesystem namesystem = cluster.getNamesystem();
final DatanodeManager dm = cluster.getNamesystem().getBlockManager(
).getDatanodeManager();
@@ -100,8 +99,9 @@ public class TestNamenodeCapacityReport {
+ " used " + used + " non DFS used " + nonDFSUsed
+ " remaining " + remaining + " perentUsed " + percentUsed
+ " percentRemaining " + percentRemaining);
-
- assertTrue(configCapacity == (used + remaining + nonDFSUsed));
+ // There will be 5% space reserved in ext filesystem which is not
+ // considered.
+ assertTrue(configCapacity >= (used + remaining + nonDFSUsed));
assertTrue(percentUsed == DFSUtilClient.getPercentUsed(used,
configCapacity));
assertTrue(percentRemaining == DFSUtilClient.getPercentRemaining(
@@ -148,7 +148,9 @@ public class TestNamenodeCapacityReport {
assertTrue(configCapacity == diskCapacity - reserved);
// Ensure new total capacity reported excludes the reserved space
- assertTrue(configCapacity == (used + remaining + nonDFSUsed));
+ // There will be 5% space reserved in ext filesystem which is not
+ // considered.
+ assertTrue(configCapacity >= (used + remaining + nonDFSUsed));
// Ensure percent used is calculated based on used and present capacity
assertTrue(percentUsed == DFSUtilClient.getPercentUsed(used,
@@ -160,9 +162,33 @@ public class TestNamenodeCapacityReport {
// Ensure percent used is calculated based on used and present capacity
assertTrue(percentRemaining == ((float)remaining * 100.0f)/(float)configCapacity);
+
+ //Adding testcase for non-dfs used where we need to consider
+ // reserved replica also.
+ final int fileCount = 5;
+ final DistributedFileSystem fs = cluster.getFileSystem();
+ // create streams and hsync to force datastreamers to start
+ DFSOutputStream[] streams = new DFSOutputStream[fileCount];
+ for (int i=0; i < fileCount; i++) {
+ streams[i] = (DFSOutputStream)fs.create(new Path("/f"+i))
+ .getWrappedStream();
+ streams[i].write("1".getBytes());
+ streams[i].hsync();
+ }
+ triggerHeartbeats(cluster.getDataNodes());
+ assertTrue(configCapacity > (namesystem.getCapacityUsed() + namesystem
+ .getCapacityRemaining() + namesystem.getNonDfsUsedSpace()));
+ // There is a chance that nonDFS usage might have slightly due to
+ // testlogs, So assume 1MB other files used within this gap
+ assertTrue(
+ (namesystem.getCapacityUsed() + namesystem.getCapacityRemaining()
+ + namesystem.getNonDfsUsedSpace() + fileCount * fs
+ .getDefaultBlockSize()) - configCapacity < 1 * 1024);
}
finally {
- if (cluster != null) {cluster.shutdown();}
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index c4a9837..9e95650 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -166,7 +166,9 @@ public class TestNameNodeMetrics {
MetricsAsserts.getLongGauge("CapacityRemaining", rb);
long capacityUsedNonDFS =
MetricsAsserts.getLongGauge("CapacityUsedNonDFS", rb);
- assert(capacityUsed + capacityRemaining + capacityUsedNonDFS ==
+ // There will be 5% space reserved in ext filesystem which is not
+ // considered.
+ assert (capacityUsed + capacityRemaining + capacityUsedNonDFS <=
capacityTotal);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/22] hadoop git commit: HADOOP-13529. Do some code refactoring.
Contributed by Genmao Yu.
Posted by sh...@apache.org.
HADOOP-13529. Do some code refactoring. Contributed by Genmao Yu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d33e928f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d33e928f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d33e928f
Branch: refs/heads/HADOOP-12756
Commit: d33e928fbeb1764a724c8f3c051bb0d8be82bbff
Parents: cdb7711
Author: Mingfei <mi...@intel.com>
Authored: Fri Aug 26 11:00:03 2016 +0800
Committer: Mingfei <mi...@intel.com>
Committed: Wed Sep 7 11:15:48 2016 +0800
----------------------------------------------------------------------
hadoop-tools/hadoop-aliyun/pom.xml | 23 +-
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 455 +++--------------
.../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 486 +++++++++++++++++++
.../fs/aliyun/oss/AliyunOSSInputStream.java | 60 +--
.../fs/aliyun/oss/AliyunOSSOutputStream.java | 129 +----
.../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 193 ++++----
.../apache/hadoop/fs/aliyun/oss/Constants.java | 3 +-
.../aliyun/oss/TestOSSFileSystemContract.java | 10 -
.../fs/aliyun/oss/TestOSSFileSystemStore.java | 121 +++++
.../fs/aliyun/oss/contract/OSSContract.java | 1 -
.../oss/contract/TestOSSContractDispCp.java | 44 ++
.../contract/TestOSSContractGetFileStatus.java | 35 ++
.../oss/contract/TestOSSContractRootDir.java | 69 +++
13 files changed, 968 insertions(+), 661 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml
index c87d13f..358b18b 100644
--- a/hadoop-tools/hadoop-aliyun/pom.xml
+++ b/hadoop-tools/hadoop-aliyun/pom.xml
@@ -128,6 +128,27 @@
<scope>test</scope>
<type>test-jar</type>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index afe7242..ad321bd 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -20,18 +20,12 @@ package org.apache.hadoop.fs.aliyun.oss;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
-import com.aliyun.oss.ClientException;
-import com.aliyun.oss.common.auth.CredentialsProvider;
-import com.aliyun.oss.common.auth.DefaultCredentialProvider;
-import com.aliyun.oss.common.auth.DefaultCredentials;
-import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -39,30 +33,13 @@ import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.aliyun.oss.AliyunOSSUtils.UserInfo;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.util.Progressable;
-import com.aliyun.oss.ClientConfiguration;
-import com.aliyun.oss.OSSClient;
-import com.aliyun.oss.OSSException;
-import com.aliyun.oss.common.comm.Protocol;
-import com.aliyun.oss.model.AbortMultipartUploadRequest;
-import com.aliyun.oss.model.CannedAccessControlList;
-import com.aliyun.oss.model.CompleteMultipartUploadRequest;
-import com.aliyun.oss.model.CompleteMultipartUploadResult;
-import com.aliyun.oss.model.CopyObjectResult;
-import com.aliyun.oss.model.DeleteObjectsRequest;
-import com.aliyun.oss.model.InitiateMultipartUploadRequest;
-import com.aliyun.oss.model.InitiateMultipartUploadResult;
-import com.aliyun.oss.model.ListObjectsRequest;
import com.aliyun.oss.model.OSSObjectSummary;
import com.aliyun.oss.model.ObjectListing;
import com.aliyun.oss.model.ObjectMetadata;
-import com.aliyun.oss.model.PartETag;
-import com.aliyun.oss.model.UploadPartCopyRequest;
-import com.aliyun.oss.model.UploadPartCopyResult;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,12 +52,8 @@ public class AliyunOSSFileSystem extends FileSystem {
LoggerFactory.getLogger(AliyunOSSFileSystem.class);
private URI uri;
private Path workingDir;
- private OSSClient ossClient;
- private String bucketName;
- private long uploadPartSize;
- private long multipartThreshold;
+ private AliyunOSSFileSystemStore store;
private int maxKeys;
- private String serverSideEncryptionAlgorithm;
@Override
public FSDataOutputStream append(Path path, int bufferSize,
@@ -91,9 +64,7 @@ public class AliyunOSSFileSystem extends FileSystem {
@Override
public void close() throws IOException {
try {
- if (ossClient != null) {
- ossClient.shutdown();
- }
+ store.close();
} finally {
super.close();
}
@@ -125,23 +96,33 @@ public class AliyunOSSFileSystem extends FileSystem {
}
return new FSDataOutputStream(new AliyunOSSOutputStream(getConf(),
- ossClient, bucketName, key, progress, statistics,
- serverSideEncryptionAlgorithm), (Statistics)(null));
+ store, key, progress, statistics), (Statistics)(null));
}
@Override
public boolean delete(Path path, boolean recursive) throws IOException {
- FileStatus status;
try {
- status = getFileStatus(path);
+ return innerDelete(getFileStatus(path), recursive);
} catch (FileNotFoundException e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Couldn't delete " + path + ": Does not exist!");
- }
+ LOG.debug("Couldn't delete {} - does not exist", path);
return false;
}
+ }
- String key = pathToKey(status.getPath());
+ /**
+ * Delete an object. See {@link #delete(Path, boolean)}.
+ *
+ * @param status fileStatus object
+ * @param recursive if path is a directory and set to
+ * true, the directory is deleted else throws an exception. In
+ * case of a file the recursive can be set to either true or false.
+ * @return true if delete is successful else false.
+ * @throws IOException due to inability to delete a directory or file.
+ */
+ private boolean innerDelete(FileStatus status, boolean recursive)
+ throws IOException {
+ Path f = status.getPath();
+ String key = pathToKey(f);
if (status.isDirectory()) {
if (!key.endsWith("/")) {
key += "/";
@@ -150,54 +131,34 @@ public class AliyunOSSFileSystem extends FileSystem {
FileStatus[] statuses = listStatus(status.getPath());
// Check whether it is an empty directory or not
if (statuses.length > 0) {
- throw new IOException("Cannot remove directory" + path +
+ throw new IOException("Cannot remove directory " + f +
": It is not empty!");
} else {
// Delete empty directory without '-r'
- ossClient.deleteObject(bucketName, key);
- statistics.incrementWriteOps(1);
+ store.deleteObject(key);
}
} else {
- ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
- listRequest.setPrefix(key);
- listRequest.setMaxKeys(maxKeys);
-
- while (true) {
- ObjectListing objects = ossClient.listObjects(listRequest);
- statistics.incrementReadOps(1);
- List<String> keysToDelete = new ArrayList<String>();
- for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
- keysToDelete.add(objectSummary.getKey());
- }
- DeleteObjectsRequest deleteRequest =
- new DeleteObjectsRequest(bucketName);
- deleteRequest.setKeys(keysToDelete);
- ossClient.deleteObjects(deleteRequest);
- statistics.incrementWriteOps(1);
- if (objects.isTruncated()) {
- listRequest.setMarker(objects.getNextMarker());
- } else {
- break;
- }
- }
+ store.deleteDirs(key);
}
} else {
- ossClient.deleteObject(bucketName, key);
- statistics.incrementWriteOps(1);
+ store.deleteObject(key);
}
- //TODO: optimize logic here
+
+ createFakeDirectoryIfNecessary(f);
+ return true;
+ }
+
+ private void createFakeDirectoryIfNecessary(Path f) throws IOException {
try {
- Path pPath = status.getPath().getParent();
+ Path pPath = f.getParent();
FileStatus pStatus = getFileStatus(pPath);
- if (pStatus.isDirectory()) {
- return true;
- } else {
+ if (pStatus.isFile()) {
throw new IOException("Path " + pPath +
" is assumed to be a directory!");
}
} catch (FileNotFoundException fnfe) {
// Make sure the parent directory exists
- return mkdir(bucketName, pathToKey(status.getPath().getParent()));
+ mkdir(pathToKey(f.getParent()));
}
}
@@ -211,22 +172,15 @@ public class AliyunOSSFileSystem extends FileSystem {
return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
}
- ObjectMetadata meta = getObjectMetadata(key);
+ ObjectMetadata meta = store.getObjectMetadata(key);
// If key not found and key does not end with "/"
if (meta == null && !key.endsWith("/")) {
// Case: dir + "/"
key += "/";
- meta = getObjectMetadata(key);
+ meta = store.getObjectMetadata(key);
}
if (meta == null) {
- // Case: dir + "/" + file
- ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
- listRequest.setPrefix(key);
- listRequest.setDelimiter("/");
- listRequest.setMaxKeys(1);
-
- ObjectListing listing = ossClient.listObjects(listRequest);
- statistics.incrementReadOps(1);
+ ObjectListing listing = store.listObjects(key, 1, "/", null);
if (!listing.getObjectSummaries().isEmpty() ||
!listing.getCommonPrefixes().isEmpty()) {
return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
@@ -242,22 +196,6 @@ public class AliyunOSSFileSystem extends FileSystem {
}
}
- /**
- * Return object metadata given object key.
- *
- * @param key object key
- * @return return null if key does not exist
- */
- private ObjectMetadata getObjectMetadata(String key) {
- try {
- return ossClient.getObjectMetadata(bucketName, key);
- } catch (OSSException osse) {
- return null;
- } finally {
- statistics.incrementReadOps(1);
- }
- }
-
@Override
public String getScheme() {
return "oss";
@@ -288,7 +226,6 @@ public class AliyunOSSFileSystem extends FileSystem {
* Initialize new FileSystem.
*
* @param name the uri of the file system, including host, port, etc.
- *
* @param conf configuration of the file system
* @throws IOException IO problems
*/
@@ -296,154 +233,16 @@ public class AliyunOSSFileSystem extends FileSystem {
super.initialize(name, conf);
uri = java.net.URI.create(name.getScheme() + "://" + name.getAuthority());
- workingDir =
- new Path("/user",
- System.getProperty("user.name")).makeQualified(uri, null);
-
- bucketName = name.getHost();
-
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS_KEY,
- MAXIMUM_CONNECTIONS_DEFAULT));
- boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS_KEY,
- SECURE_CONNECTIONS_DEFAULT);
- clientConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
- clientConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES_KEY,
- MAX_ERROR_RETRIES_DEFAULT));
- clientConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT_KEY,
- ESTABLISH_TIMEOUT_DEFAULT));
- clientConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT_KEY,
- SOCKET_TIMEOUT_DEFAULT));
-
- String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, "");
- int proxyPort = conf.getInt(PROXY_PORT_KEY, -1);
- if (!proxyHost.isEmpty()) {
- clientConf.setProxyHost(proxyHost);
- if (proxyPort >= 0) {
- clientConf.setProxyPort(proxyPort);
- } else {
- if (secureConnections) {
- LOG.warn("Proxy host set without port. Using HTTPS default 443");
- clientConf.setProxyPort(443);
- } else {
- LOG.warn("Proxy host set without port. Using HTTP default 80");
- clientConf.setProxyPort(80);
- }
- }
- String proxyUsername = conf.getTrimmed(PROXY_USERNAME_KEY);
- String proxyPassword = conf.getTrimmed(PROXY_PASSWORD_KEY);
- if ((proxyUsername == null) != (proxyPassword == null)) {
- String msg = "Proxy error: " + PROXY_USERNAME_KEY + " or " +
- PROXY_PASSWORD_KEY + " set without the other.";
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- clientConf.setProxyUsername(proxyUsername);
- clientConf.setProxyPassword(proxyPassword);
- clientConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN_KEY));
- clientConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION_KEY));
- } else if (proxyPort >= 0) {
- String msg = "Proxy error: " + PROXY_PORT_KEY + " set without " +
- PROXY_HOST_KEY;
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
-
- String endPoint = conf.getTrimmed(ENDPOINT_KEY, "");
- ossClient =
- new OSSClient(endPoint, getCredentialsProvider(name, conf), clientConf);
+ workingDir = new Path("/user",
+ System.getProperty("user.name")).makeQualified(uri, null);
+ store = new AliyunOSSFileSystemStore();
+ store.initialize(name, conf, statistics);
maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
- uploadPartSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
- MULTIPART_UPLOAD_SIZE_DEFAULT);
- multipartThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
- MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
-
- if (uploadPartSize < 5 * 1024 * 1024) {
- LOG.warn(MULTIPART_UPLOAD_SIZE_KEY + " must be at least 5 MB");
- uploadPartSize = 5 * 1024 * 1024;
- }
-
- if (multipartThreshold < 5 * 1024 * 1024) {
- LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be at least 5 MB");
- multipartThreshold = 5 * 1024 * 1024;
- }
-
- if (multipartThreshold > 1024 * 1024 * 1024) {
- LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be less than 1 GB");
- multipartThreshold = 1024 * 1024 * 1024;
- }
-
- String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT);
- if (!cannedACLName.isEmpty()) {
- CannedAccessControlList cannedACL =
- CannedAccessControlList.valueOf(cannedACLName);
- ossClient.setBucketAcl(bucketName, cannedACL);
- }
-
- serverSideEncryptionAlgorithm =
- conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY, "");
-
setConf(conf);
}
/**
- * Create the default credential provider, or load in one explicitly
- * identified in the configuration.
- * @param name the uri of the file system
- * @param conf configuration
- * @return a credential provider
- * @throws IOException on any problem. Class construction issues may be
- * nested inside the IOE.
- */
- private CredentialsProvider getCredentialsProvider(URI name,
- Configuration conf) throws IOException {
- CredentialsProvider credentials;
-
- String className = conf.getTrimmed(ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY);
- if (StringUtils.isEmpty(className)) {
- Configuration newConf =
- ProviderUtils.excludeIncompatibleCredentialProviders(conf,
- AliyunOSSFileSystem.class);
- String accessKey =
- AliyunOSSUtils.getPassword(newConf, ACCESS_KEY,
- UserInfo.EMPTY.getUser());
- String secretKey =
- AliyunOSSUtils.getPassword(newConf, SECRET_KEY,
- UserInfo.EMPTY.getPassword());
- credentials =
- new DefaultCredentialProvider(
- new DefaultCredentials(accessKey, secretKey));
-
- } else {
- try {
- LOG.debug("Credential provider class is:" + className);
- Class<?> credClass = Class.forName(className);
- try {
- credentials =
- (CredentialsProvider)credClass.getDeclaredConstructor(
- URI.class, Configuration.class).newInstance(this.uri, conf);
- } catch (NoSuchMethodException | SecurityException e) {
- credentials =
- (CredentialsProvider)credClass.getDeclaredConstructor()
- .newInstance();
- }
- } catch (ClassNotFoundException e) {
- throw new IOException(className + " not found.", e);
- } catch (NoSuchMethodException | SecurityException e) {
- throw new IOException(String.format("%s constructor exception. A " +
- "class specified in %s must provide an accessible constructor " +
- "accepting URI and Configuration, or an accessible default " +
- "constructor.", className, ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY), e);
- } catch (ReflectiveOperationException | IllegalArgumentException e) {
- throw new IOException(className + " instantiation exception.", e);
- }
- }
-
- return credentials;
- }
-
- /**
* Check if OSS object represents a directory.
*
* @param name object key
@@ -456,10 +255,10 @@ public class AliyunOSSFileSystem extends FileSystem {
}
/**
- * Turns a path (relative or otherwise) into an OSS key.
+ * Turn a path (relative or otherwise) into an OSS key.
*
- * @param path the path of the file
- * @return the key of the object that represent the file
+ * @param path the path of the file.
+ * @return the key of the object that represents the file.
*/
private String pathToKey(Path path) {
if (!path.isAbsolute()) {
@@ -492,18 +291,12 @@ public class AliyunOSSFileSystem extends FileSystem {
key = key + "/";
}
- ListObjectsRequest listObjectsRequest =
- new ListObjectsRequest(bucketName);
- listObjectsRequest.setPrefix(key);
- listObjectsRequest.setDelimiter("/");
- listObjectsRequest.setMaxKeys(maxKeys);
-
if (LOG.isDebugEnabled()) {
LOG.debug("listStatus: doing listObjects for directory " + key);
}
+ ObjectListing objects = store.listObjects(key, maxKeys, "/", null);
while (true) {
- ObjectListing objects = ossClient.listObjects(listObjectsRequest);
statistics.incrementReadOps(1);
for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
Path keyPath = keyToPath(objectSummary.getKey())
@@ -539,7 +332,8 @@ public class AliyunOSSFileSystem extends FileSystem {
if (LOG.isDebugEnabled()) {
LOG.debug("listStatus: list truncated - getting next batch");
}
- listObjectsRequest.setMarker(objects.getNextMarker());
+ objects = store.listObjects(key, maxKeys, "/",
+ objects.getNextMarker());
statistics.incrementReadOps(1);
} else {
break;
@@ -558,27 +352,17 @@ public class AliyunOSSFileSystem extends FileSystem {
/**
* Used to create an empty file that represents an empty directory.
*
- * @param bucket the bucket this directory belongs to
* @param key directory path
- * @return true if directory successfully created
+ * @return true if directory is successfully created
* @throws IOException
*/
- private boolean mkdir(final String bucket, final String key)
- throws IOException {
+ private boolean mkdir(final String key) throws IOException {
String dirName = key;
- ObjectMetadata dirMeta = new ObjectMetadata();
- byte[] buffer = new byte[0];
- ByteArrayInputStream in = new ByteArrayInputStream(buffer);
- dirMeta.setContentLength(0);
if (!key.endsWith("/")) {
dirName += "/";
}
- try {
- ossClient.putObject(bucket, dirName, in, dirMeta);
- return true;
- } finally {
- in.close();
- }
+ store.storeEmptyFile(dirName);
+ return true;
}
@Override
@@ -595,14 +379,14 @@ public class AliyunOSSFileSystem extends FileSystem {
} catch (FileNotFoundException e) {
validatePath(path);
String key = pathToKey(path);
- return mkdir(bucketName, key);
+ return mkdir(key);
}
}
/**
* Check whether the path is a valid path.
*
- * @param path the path to be checked
+ * @param path the path to be checked.
* @throws IOException
*/
private void validatePath(Path path) throws IOException {
@@ -631,8 +415,8 @@ public class AliyunOSSFileSystem extends FileSystem {
" because it is a directory");
}
- return new FSDataInputStream(new AliyunOSSInputStream(getConf(), ossClient,
- bucketName, pathToKey(path), fileStatus.getLen(), statistics));
+ return new FSDataInputStream(new AliyunOSSInputStream(getConf(), store,
+ pathToKey(path), fileStatus.getLen(), statistics));
}
@Override
@@ -696,126 +480,31 @@ public class AliyunOSSFileSystem extends FileSystem {
} else {
copyFile(srcPath, dstPath);
}
- if (srcPath.equals(dstPath)) {
- return true;
- } else {
- return delete(srcPath, true);
- }
+
+ return srcPath.equals(dstPath) || delete(srcPath, true);
}
/**
* Copy file from source path to destination path.
- * (the caller should make sure srcPath is a file and dstPath is valid.)
+ * (the caller should make sure srcPath is a file and dstPath is valid)
*
- * @param srcPath source path
- * @param dstPath destination path
- * @return true if successfully copied
+ * @param srcPath source path.
+ * @param dstPath destination path.
+ * @return true if file is successfully copied.
*/
private boolean copyFile(Path srcPath, Path dstPath) {
String srcKey = pathToKey(srcPath);
String dstKey = pathToKey(dstPath);
- return copyFile(srcKey, dstKey);
- }
-
- /**
- * Copy an object from source key to destination key.
- *
- * @param srcKey source key
- * @param dstKey destination key
- * @return true if successfully copied
- */
- private boolean copyFile(String srcKey, String dstKey) {
- ObjectMetadata objectMeta =
- ossClient.getObjectMetadata(bucketName, srcKey);
- long dataLen = objectMeta.getContentLength();
- if (dataLen <= multipartThreshold) {
- return singleCopy(srcKey, dstKey);
- } else {
- return multipartCopy(srcKey, dataLen, dstKey);
- }
- }
-
- /**
- * Use single copy to copy an oss object.
- *
- * @param srcKey source key
- * @param dstKey destination key
- * @return true if successfully copied
- * (the caller should make sure srcPath is a file and dstPath is valid)
- */
- private boolean singleCopy(String srcKey, String dstKey) {
- CopyObjectResult copyResult =
- ossClient.copyObject(bucketName, srcKey, bucketName, dstKey);
- LOG.debug(copyResult.getETag());
- return true;
- }
-
- /**
- * Use multipart copy to copy an oss object.
- * (the caller should make sure srcPath is a file and dstPath is valid)
- *
- * @param srcKey source key
- * @param dataLen data size of the object to copy
- * @param dstKey destination key
- * @return true if successfully copied, or false if upload is aborted
- */
- private boolean multipartCopy(String srcKey, long dataLen, String dstKey) {
- int partNum = (int)(dataLen / uploadPartSize);
- if (dataLen % uploadPartSize != 0) {
- partNum++;
- }
- InitiateMultipartUploadRequest initiateMultipartUploadRequest =
- new InitiateMultipartUploadRequest(bucketName, dstKey);
- ObjectMetadata meta = new ObjectMetadata();
- if (!serverSideEncryptionAlgorithm.isEmpty()) {
- meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
- }
- initiateMultipartUploadRequest.setObjectMetadata(meta);
- InitiateMultipartUploadResult initiateMultipartUploadResult =
- ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
- String uploadId = initiateMultipartUploadResult.getUploadId();
- List<PartETag> partETags = new ArrayList<PartETag>();
- try {
- for (int i = 0; i < partNum; i++) {
- long skipBytes = uploadPartSize * i;
- long size = (uploadPartSize < dataLen - skipBytes) ?
- uploadPartSize : dataLen - skipBytes;
- UploadPartCopyRequest partCopyRequest = new UploadPartCopyRequest();
- partCopyRequest.setSourceBucketName(bucketName);
- partCopyRequest.setSourceKey(srcKey);
- partCopyRequest.setBucketName(bucketName);
- partCopyRequest.setKey(dstKey);
- partCopyRequest.setUploadId(uploadId);
- partCopyRequest.setPartSize(size);
- partCopyRequest.setBeginIndex(skipBytes);
- partCopyRequest.setPartNumber(i + 1);
- UploadPartCopyResult partCopyResult =
- ossClient.uploadPartCopy(partCopyRequest);
- statistics.incrementWriteOps(1);
- partETags.add(partCopyResult.getPartETag());
- }
- CompleteMultipartUploadRequest completeMultipartUploadRequest =
- new CompleteMultipartUploadRequest(bucketName, dstKey,
- uploadId, partETags);
- CompleteMultipartUploadResult completeMultipartUploadResult =
- ossClient.completeMultipartUpload(completeMultipartUploadRequest);
- LOG.debug(completeMultipartUploadResult.getETag());
- return true;
- } catch (OSSException | ClientException e) {
- AbortMultipartUploadRequest abortMultipartUploadRequest =
- new AbortMultipartUploadRequest(bucketName, dstKey, uploadId);
- ossClient.abortMultipartUpload(abortMultipartUploadRequest);
- return false;
- }
+ return store.copyFile(srcKey, dstKey);
}
/**
* Copy a directory from source path to destination path.
* (the caller should make sure srcPath is a directory, and dstPath is valid)
*
- * @param srcPath source path
- * @param dstPath destination path
- * @return true if successfully copied
+ * @param srcPath source path.
+ * @param dstPath destination path.
+ * @return true if directory is successfully copied.
*/
private boolean copyDirectory(Path srcPath, Path dstPath) {
String srcKey = pathToKey(srcPath);
@@ -835,21 +524,18 @@ public class AliyunOSSFileSystem extends FileSystem {
return false;
}
- ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName);
- listObjectsRequest.setPrefix(srcKey);
- listObjectsRequest.setMaxKeys(maxKeys);
-
- ObjectListing objects = ossClient.listObjects(listObjectsRequest);
+ ObjectListing objects = store.listObjects(srcKey, maxKeys, null, null);
statistics.incrementReadOps(1);
// Copy files from src folder to dst
while (true) {
for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
String newKey =
dstKey.concat(objectSummary.getKey().substring(srcKey.length()));
- copyFile(objectSummary.getKey(), newKey);
+ store.copyFile(objectSummary.getKey(), newKey);
}
if (objects.isTruncated()) {
- listObjectsRequest.setMarker(objects.getNextMarker());
+ objects = store.listObjects(srcKey, maxKeys, null,
+ objects.getNextMarker());
statistics.incrementReadOps(1);
} else {
break;
@@ -863,4 +549,7 @@ public class AliyunOSSFileSystem extends FileSystem {
this.workingDir = dir;
}
+ public AliyunOSSFileSystemStore getStore() {
+ return store;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
new file mode 100644
index 0000000..b3cd1bd
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -0,0 +1,486 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.aliyun.oss;
+
+import com.aliyun.oss.ClientConfiguration;
+import com.aliyun.oss.ClientException;
+import com.aliyun.oss.OSSClient;
+import com.aliyun.oss.OSSException;
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.aliyun.oss.common.comm.Protocol;
+import com.aliyun.oss.model.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+/**
+ * Core implementation of Aliyun OSS Filesystem for Hadoop.
+ * Provides the bridging logic between Hadoop's abstract filesystem and
+ * Aliyun OSS.
+ */
+public class AliyunOSSFileSystemStore {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(AliyunOSSFileSystemStore.class);
+ private FileSystem.Statistics statistics;
+ private OSSClient ossClient;
+ private String bucketName;
+ private long uploadPartSize;
+ private long multipartThreshold;
+ private long partSize;
+ private int maxKeys;
+ private String serverSideEncryptionAlgorithm;
+
+ public void initialize(URI uri, Configuration conf,
+ FileSystem.Statistics stat) throws IOException {
+ statistics = stat;
+ ClientConfiguration clientConf = new ClientConfiguration();
+ clientConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS_KEY,
+ MAXIMUM_CONNECTIONS_DEFAULT));
+ boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS_KEY,
+ SECURE_CONNECTIONS_DEFAULT);
+ clientConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
+ clientConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES_KEY,
+ MAX_ERROR_RETRIES_DEFAULT));
+ clientConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT_KEY,
+ ESTABLISH_TIMEOUT_DEFAULT));
+ clientConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT_KEY,
+ SOCKET_TIMEOUT_DEFAULT));
+
+ String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, "");
+ int proxyPort = conf.getInt(PROXY_PORT_KEY, -1);
+ if (!proxyHost.isEmpty()) {
+ clientConf.setProxyHost(proxyHost);
+ if (proxyPort >= 0) {
+ clientConf.setProxyPort(proxyPort);
+ } else {
+ if (secureConnections) {
+ LOG.warn("Proxy host set without port. Using HTTPS default 443");
+ clientConf.setProxyPort(443);
+ } else {
+ LOG.warn("Proxy host set without port. Using HTTP default 80");
+ clientConf.setProxyPort(80);
+ }
+ }
+ String proxyUsername = conf.getTrimmed(PROXY_USERNAME_KEY);
+ String proxyPassword = conf.getTrimmed(PROXY_PASSWORD_KEY);
+ if ((proxyUsername == null) != (proxyPassword == null)) {
+ String msg = "Proxy error: " + PROXY_USERNAME_KEY + " or " +
+ PROXY_PASSWORD_KEY + " set without the other.";
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+ clientConf.setProxyUsername(proxyUsername);
+ clientConf.setProxyPassword(proxyPassword);
+ clientConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN_KEY));
+ clientConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION_KEY));
+ } else if (proxyPort >= 0) {
+ String msg = "Proxy error: " + PROXY_PORT_KEY + " set without " +
+ PROXY_HOST_KEY;
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+
+ String endPoint = conf.getTrimmed(ENDPOINT_KEY, "");
+ CredentialsProvider provider =
+ AliyunOSSUtils.getCredentialsProvider(uri, conf);
+ ossClient = new OSSClient(endPoint, provider, clientConf);
+ uploadPartSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
+ MULTIPART_UPLOAD_SIZE_DEFAULT);
+ multipartThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
+ MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
+ partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
+ MULTIPART_UPLOAD_SIZE_DEFAULT);
+ if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) {
+ partSize = MIN_MULTIPART_UPLOAD_PART_SIZE;
+ }
+ serverSideEncryptionAlgorithm =
+ conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY, "");
+
+ if (uploadPartSize < 5 * 1024 * 1024) {
+ LOG.warn(MULTIPART_UPLOAD_SIZE_KEY + " must be at least 5 MB");
+ uploadPartSize = 5 * 1024 * 1024;
+ }
+
+ if (multipartThreshold < 5 * 1024 * 1024) {
+ LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be at least 5 MB");
+ multipartThreshold = 5 * 1024 * 1024;
+ }
+
+ if (multipartThreshold > 1024 * 1024 * 1024) {
+ LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be less than 1 GB");
+ multipartThreshold = 1024 * 1024 * 1024;
+ }
+
+ String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT);
+ if (!cannedACLName.isEmpty()) {
+ CannedAccessControlList cannedACL =
+ CannedAccessControlList.valueOf(cannedACLName);
+ ossClient.setBucketAcl(bucketName, cannedACL);
+ }
+
+ maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
+ bucketName = uri.getHost();
+ }
+
+ /**
+ * Delete an object, and update write operation statistics.
+ *
+ * @param key key to blob to delete.
+ */
+ public void deleteObject(String key) {
+ ossClient.deleteObject(bucketName, key);
+ statistics.incrementWriteOps(1);
+ }
+
+ /**
+ * Delete a list of keys, and update write operation statistics.
+ *
+ * @param keysToDelete collection of keys to delete.
+ */
+ public void deleteObjects(List<String> keysToDelete) {
+ DeleteObjectsRequest deleteRequest =
+ new DeleteObjectsRequest(bucketName);
+ deleteRequest.setKeys(keysToDelete);
+ ossClient.deleteObjects(deleteRequest);
+ statistics.incrementWriteOps(keysToDelete.size());
+ }
+
+ /**
+ * Delete a directory from Aliyun OSS.
+ *
+ * @param key directory key to delete.
+ */
+ public void deleteDirs(String key) {
+ ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
+ listRequest.setPrefix(key);
+ listRequest.setMaxKeys(maxKeys);
+
+ while (true) {
+ ObjectListing objects = ossClient.listObjects(listRequest);
+ statistics.incrementReadOps(1);
+ List<String> keysToDelete = new ArrayList<String>();
+ for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
+ keysToDelete.add(objectSummary.getKey());
+ }
+ deleteObjects(keysToDelete);
+ if (objects.isTruncated()) {
+ listRequest.setMarker(objects.getNextMarker());
+ } else {
+ break;
+ }
+ }
+ }
+
+ /**
+ * Return metadata of a given object key.
+ *
+ * @param key object key.
+ * @return return null if key does not exist.
+ */
+ public ObjectMetadata getObjectMetadata(String key) {
+ try {
+ return ossClient.getObjectMetadata(bucketName, key);
+ } catch (OSSException osse) {
+ return null;
+ } finally {
+ statistics.incrementReadOps(1);
+ }
+ }
+
+ /**
+ * Upload an empty file as an OSS object, using single upload.
+ *
+ * @param key object key.
+ * @throws IOException if failed to upload object.
+ */
+ public void storeEmptyFile(String key) throws IOException {
+ ObjectMetadata dirMeta = new ObjectMetadata();
+ byte[] buffer = new byte[0];
+ ByteArrayInputStream in = new ByteArrayInputStream(buffer);
+ dirMeta.setContentLength(0);
+ try {
+ ossClient.putObject(bucketName, key, in, dirMeta);
+ } finally {
+ in.close();
+ }
+ }
+
+ /**
+ * Copy an object from source key to destination key.
+ *
+ * @param srcKey source key.
+ * @param dstKey destination key.
+ * @return true if file is successfully copied.
+ */
+ public boolean copyFile(String srcKey, String dstKey) {
+ ObjectMetadata objectMeta =
+ ossClient.getObjectMetadata(bucketName, srcKey);
+ long contentLength = objectMeta.getContentLength();
+ if (contentLength <= multipartThreshold) {
+ return singleCopy(srcKey, dstKey);
+ } else {
+ return multipartCopy(srcKey, contentLength, dstKey);
+ }
+ }
+
+ /**
+ * Use single copy to copy an OSS object.
+ * (The caller should make sure srcPath is a file and dstPath is valid)
+ *
+ * @param srcKey source key.
+ * @param dstKey destination key.
+ * @return true if object is successfully copied.
+ */
+ private boolean singleCopy(String srcKey, String dstKey) {
+ CopyObjectResult copyResult =
+ ossClient.copyObject(bucketName, srcKey, bucketName, dstKey);
+ LOG.debug(copyResult.getETag());
+ return true;
+ }
+
+ /**
+ * Use multipart copy to copy an OSS object.
+ * (The caller should make sure srcPath is a file and dstPath is valid)
+ *
+ * @param srcKey source key.
+ * @param contentLength data size of the object to copy.
+ * @param dstKey destination key.
+ * @return true if success, or false if upload is aborted.
+ */
+ private boolean multipartCopy(String srcKey, long contentLength,
+ String dstKey) {
+ long realPartSize =
+ AliyunOSSUtils.calculatePartSize(contentLength, uploadPartSize);
+ int partNum = (int) (contentLength / realPartSize);
+ if (contentLength % realPartSize != 0) {
+ partNum++;
+ }
+ InitiateMultipartUploadRequest initiateMultipartUploadRequest =
+ new InitiateMultipartUploadRequest(bucketName, dstKey);
+ ObjectMetadata meta = new ObjectMetadata();
+ if (!serverSideEncryptionAlgorithm.isEmpty()) {
+ meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
+ }
+ initiateMultipartUploadRequest.setObjectMetadata(meta);
+ InitiateMultipartUploadResult initiateMultipartUploadResult =
+ ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
+ String uploadId = initiateMultipartUploadResult.getUploadId();
+ List<PartETag> partETags = new ArrayList<PartETag>();
+ try {
+ for (int i = 0; i < partNum; i++) {
+ long skipBytes = realPartSize * i;
+ long size = (realPartSize < contentLength - skipBytes) ?
+ realPartSize : contentLength - skipBytes;
+ UploadPartCopyRequest partCopyRequest = new UploadPartCopyRequest();
+ partCopyRequest.setSourceBucketName(bucketName);
+ partCopyRequest.setSourceKey(srcKey);
+ partCopyRequest.setBucketName(bucketName);
+ partCopyRequest.setKey(dstKey);
+ partCopyRequest.setUploadId(uploadId);
+ partCopyRequest.setPartSize(size);
+ partCopyRequest.setBeginIndex(skipBytes);
+ partCopyRequest.setPartNumber(i + 1);
+ UploadPartCopyResult partCopyResult =
+ ossClient.uploadPartCopy(partCopyRequest);
+ statistics.incrementWriteOps(1);
+ partETags.add(partCopyResult.getPartETag());
+ }
+ CompleteMultipartUploadRequest completeMultipartUploadRequest =
+ new CompleteMultipartUploadRequest(bucketName, dstKey,
+ uploadId, partETags);
+ CompleteMultipartUploadResult completeMultipartUploadResult =
+ ossClient.completeMultipartUpload(completeMultipartUploadRequest);
+ LOG.debug(completeMultipartUploadResult.getETag());
+ return true;
+ } catch (OSSException | ClientException e) {
+ AbortMultipartUploadRequest abortMultipartUploadRequest =
+ new AbortMultipartUploadRequest(bucketName, dstKey, uploadId);
+ ossClient.abortMultipartUpload(abortMultipartUploadRequest);
+ return false;
+ }
+ }
+
+ /**
+ * Upload a file as an OSS object, using single upload.
+ *
+ * @param key object key.
+ * @param file local file to upload.
+ * @throws IOException if failed to upload object.
+ */
+ public void uploadObject(String key, File file) throws IOException {
+ File object = file.getAbsoluteFile();
+ FileInputStream fis = new FileInputStream(object);
+ ObjectMetadata meta = new ObjectMetadata();
+ meta.setContentLength(object.length());
+ if (!serverSideEncryptionAlgorithm.isEmpty()) {
+ meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
+ }
+ try {
+ PutObjectResult result = ossClient.putObject(bucketName, key, fis, meta);
+ LOG.debug(result.getETag());
+ statistics.incrementWriteOps(1);
+ } finally {
+ fis.close();
+ }
+ }
+
+ /**
+ * Upload a file as an OSS object, using multipart upload.
+ *
+ * @param key object key.
+ * @param file local file to upload.
+ * @throws IOException if failed to upload object.
+ */
+ public void multipartUploadObject(String key, File file) throws IOException {
+ File object = file.getAbsoluteFile();
+ long dataLen = object.length();
+ long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize);
+ int partNum = (int) (dataLen / realPartSize);
+ if (dataLen % realPartSize != 0) {
+ partNum += 1;
+ }
+
+ InitiateMultipartUploadRequest initiateMultipartUploadRequest =
+ new InitiateMultipartUploadRequest(bucketName, key);
+ ObjectMetadata meta = new ObjectMetadata();
+ if (!serverSideEncryptionAlgorithm.isEmpty()) {
+ meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
+ }
+ initiateMultipartUploadRequest.setObjectMetadata(meta);
+ InitiateMultipartUploadResult initiateMultipartUploadResult =
+ ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
+ List<PartETag> partETags = new ArrayList<PartETag>();
+ String uploadId = initiateMultipartUploadResult.getUploadId();
+
+ try {
+ for (int i = 0; i < partNum; i++) {
+ // TODO: Optimize this, avoid opening the object multiple times
+ FileInputStream fis = new FileInputStream(object);
+ try {
+ long skipBytes = realPartSize * i;
+ AliyunOSSUtils.skipFully(fis, skipBytes);
+ long size = (realPartSize < dataLen - skipBytes) ?
+ realPartSize : dataLen - skipBytes;
+ UploadPartRequest uploadPartRequest = new UploadPartRequest();
+ uploadPartRequest.setBucketName(bucketName);
+ uploadPartRequest.setKey(key);
+ uploadPartRequest.setUploadId(uploadId);
+ uploadPartRequest.setInputStream(fis);
+ uploadPartRequest.setPartSize(size);
+ uploadPartRequest.setPartNumber(i + 1);
+ UploadPartResult uploadPartResult =
+ ossClient.uploadPart(uploadPartRequest);
+ statistics.incrementWriteOps(1);
+ partETags.add(uploadPartResult.getPartETag());
+ } finally {
+ fis.close();
+ }
+ }
+ CompleteMultipartUploadRequest completeMultipartUploadRequest =
+ new CompleteMultipartUploadRequest(bucketName, key,
+ uploadId, partETags);
+ CompleteMultipartUploadResult completeMultipartUploadResult =
+ ossClient.completeMultipartUpload(completeMultipartUploadRequest);
+ LOG.debug(completeMultipartUploadResult.getETag());
+ } catch (OSSException | ClientException e) {
+ AbortMultipartUploadRequest abortMultipartUploadRequest =
+ new AbortMultipartUploadRequest(bucketName, key, uploadId);
+ ossClient.abortMultipartUpload(abortMultipartUploadRequest);
+ }
+ }
+
+ /**
+ * list objects.
+ *
+ * @param prefix prefix.
+ * @param maxListingLength max no. of entries
+ * @param delimiter delimiter.
+ * @param marker last key in any previous search.
+ * @return a list of matches.
+ */
+ public ObjectListing listObjects(String prefix, int maxListingLength,
+ String delimiter, String marker) {
+ ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
+ listRequest.setPrefix(prefix);
+ listRequest.setDelimiter(delimiter);
+ listRequest.setMaxKeys(maxListingLength);
+ listRequest.setMarker(marker);
+
+ ObjectListing listing = ossClient.listObjects(listRequest);
+ statistics.incrementReadOps(1);
+ return listing;
+ }
+
+ /**
+ * Retrieve a part of an object.
+ *
+ * @param key the object name that is being retrieved from the Aliyun OSS.
+ * @param byteStart start position.
+ * @param byteEnd end position.
+ * @return This method returns null if the key is not found.
+ */
+ public InputStream retrieve(String key, long byteStart, long byteEnd) {
+ try {
+ GetObjectRequest request = new GetObjectRequest(bucketName, key);
+ request.setRange(byteStart, byteEnd);
+ return ossClient.getObject(request).getObjectContent();
+ } catch (OSSException | ClientException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Close OSS client properly.
+ */
+ public void close() {
+ if (ossClient != null) {
+ ossClient.shutdown();
+ ossClient = null;
+ }
+ }
+
+ /**
+ * Clean up all objects matching the prefix.
+ *
+ * @param prefix Aliyun OSS object prefix.
+ */
+ public void purge(String prefix) {
+ String key;
+ try {
+ ObjectListing objects = listObjects(prefix, maxKeys, null, null);
+ for (OSSObjectSummary object : objects.getObjectSummaries()) {
+ key = object.getKey();
+ ossClient.deleteObject(bucketName, key);
+ }
+
+ for (String dir: objects.getCommonPrefixes()) {
+ deleteDirs(dir);
+ }
+ } catch (OSSException | ClientException e) {
+ LOG.error("Failed to purge " + prefix);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
index b12e3f0..69265fb 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
@@ -27,12 +27,10 @@ import java.io.InputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileSystem.Statistics;
-import com.aliyun.oss.OSSClient;
-import com.aliyun.oss.model.GetObjectRequest;
-
/**
* The input stream for OSS blob system.
* The class uses multi-part downloading to read data from the object content
@@ -40,27 +38,23 @@ import com.aliyun.oss.model.GetObjectRequest;
*/
public class AliyunOSSInputStream extends FSInputStream {
public static final Log LOG = LogFactory.getLog(AliyunOSSInputStream.class);
- private static final int MAX_RETRIES = 10;
private final long downloadPartSize;
-
- private String bucketName;
- private String key;
- private OSSClient ossClient;
+ private AliyunOSSFileSystemStore store;
+ private final String key;
private Statistics statistics;
private boolean closed;
private InputStream wrappedStream = null;
- private long dataLen;
+ private long contentLength;
private long position;
private long partRemaining;
- public AliyunOSSInputStream(Configuration conf, OSSClient client,
- String bucketName, String key, Long dataLen, Statistics statistics)
- throws IOException {
- this.bucketName = bucketName;
+ public AliyunOSSInputStream(Configuration conf,
+ AliyunOSSFileSystemStore store, String key, Long contentLength,
+ Statistics statistics) throws IOException {
+ this.store = store;
this.key = key;
- ossClient = client;
this.statistics = statistics;
- this.dataLen = dataLen;
+ this.contentLength = contentLength;
downloadPartSize = conf.getLong(MULTIPART_DOWNLOAD_SIZE_KEY,
MULTIPART_DOWNLOAD_SIZE_DEFAULT);
reopen(0);
@@ -75,18 +69,17 @@ public class AliyunOSSInputStream extends FSInputStream {
* @throws IOException if failed to reopen
*/
private synchronized void reopen(long pos) throws IOException {
-
- long partLen;
+ long partSize;
if (pos < 0) {
- throw new EOFException("Cannot seek at negtive position:" + pos);
- } else if (pos > dataLen) {
- throw new EOFException("Cannot seek after EOF, fileLen:" + dataLen +
- " position:" + pos);
- } else if (pos + downloadPartSize > dataLen) {
- partLen = dataLen - pos;
+ throw new EOFException("Cannot seek at negative position:" + pos);
+ } else if (pos > contentLength) {
+ throw new EOFException("Cannot seek after EOF, contentLength:" +
+ contentLength + " position:" + pos);
+ } else if (pos + downloadPartSize > contentLength) {
+ partSize = contentLength - pos;
} else {
- partLen = downloadPartSize;
+ partSize = downloadPartSize;
}
if (wrappedStream != null) {
@@ -96,21 +89,19 @@ public class AliyunOSSInputStream extends FSInputStream {
wrappedStream.close();
}
- GetObjectRequest request = new GetObjectRequest(bucketName, key);
- request.setRange(pos, pos + partLen - 1);
- wrappedStream = ossClient.getObject(request).getObjectContent();
+ wrappedStream = store.retrieve(key, pos, pos + partSize -1);
if (wrappedStream == null) {
throw new IOException("Null IO stream");
}
position = pos;
- partRemaining = partLen;
+ partRemaining = partSize;
}
@Override
public synchronized int read() throws IOException {
checkNotClosed();
- if (partRemaining <= 0 && position < dataLen) {
+ if (partRemaining <= 0 && position < contentLength) {
reopen(position);
}
@@ -139,13 +130,14 @@ public class AliyunOSSInputStream extends FSInputStream {
/**
- * Check whether the input stream is closed.
+ * Verify that the input stream is open. Non blocking; this gives
+ * the last state of the volatile {@link #closed} field.
*
- * @throws IOException if stream is closed
+ * @throws IOException if the connection is closed.
*/
private void checkNotClosed() throws IOException {
if (closed) {
- throw new IOException("Stream is closed!");
+ throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
}
}
@@ -164,7 +156,7 @@ public class AliyunOSSInputStream extends FSInputStream {
int bytesRead = 0;
// Not EOF, and read not done
- while (position < dataLen && bytesRead < len) {
+ while (position < contentLength && bytesRead < len) {
if (partRemaining == 0) {
reopen(position);
}
@@ -219,7 +211,7 @@ public class AliyunOSSInputStream extends FSInputStream {
public synchronized int available() throws IOException {
checkNotClosed();
- long remaining = dataLen - position;
+ long remaining = contentLength - position;
if (remaining > Integer.MAX_VALUE) {
return Integer.MAX_VALUE;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
index 1e16df9..c952d0a 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
@@ -22,15 +22,10 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
import java.io.BufferedOutputStream;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-import com.aliyun.oss.ClientException;
-import com.aliyun.oss.OSSException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -38,18 +33,6 @@ import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.util.Progressable;
-import com.aliyun.oss.OSSClient;
-import com.aliyun.oss.model.AbortMultipartUploadRequest;
-import com.aliyun.oss.model.CompleteMultipartUploadRequest;
-import com.aliyun.oss.model.CompleteMultipartUploadResult;
-import com.aliyun.oss.model.InitiateMultipartUploadRequest;
-import com.aliyun.oss.model.InitiateMultipartUploadResult;
-import com.aliyun.oss.model.ObjectMetadata;
-import com.aliyun.oss.model.PartETag;
-import com.aliyun.oss.model.PutObjectResult;
-import com.aliyun.oss.model.UploadPartRequest;
-import com.aliyun.oss.model.UploadPartResult;
-
/**
* The output stream for OSS blob system.
* Data will be buffered on local disk, then uploaded to OSS in
@@ -57,36 +40,24 @@ import com.aliyun.oss.model.UploadPartResult;
*/
public class AliyunOSSOutputStream extends OutputStream {
public static final Log LOG = LogFactory.getLog(AliyunOSSOutputStream.class);
- private String bucketName;
- private String key;
+ private AliyunOSSFileSystemStore store;
+ private final String key;
private Statistics statistics;
private Progressable progress;
- private String serverSideEncryptionAlgorithm;
- private long partSize;
private long partSizeThreshold;
private LocalDirAllocator dirAlloc;
private boolean closed;
private File tmpFile;
private BufferedOutputStream backupStream;
- private OSSClient ossClient;
- public AliyunOSSOutputStream(Configuration conf, OSSClient client,
- String bucketName, String key, Progressable progress,
- Statistics statistics, String serverSideEncryptionAlgorithm)
- throws IOException {
- this.bucketName = bucketName;
+ public AliyunOSSOutputStream(Configuration conf,
+ AliyunOSSFileSystemStore store, String key, Progressable progress,
+ Statistics statistics) throws IOException {
+ this.store = store;
this.key = key;
// The caller cann't get any progress information
this.progress = progress;
- ossClient = client;
this.statistics = statistics;
- this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm;
-
- partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
- MULTIPART_UPLOAD_SIZE_DEFAULT);
- if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) {
- partSize = MIN_MULTIPART_UPLOAD_PART_SIZE;
- }
partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
@@ -113,9 +84,9 @@ public class AliyunOSSOutputStream extends OutputStream {
long dataLen = tmpFile.length();
try {
if (dataLen <= partSizeThreshold) {
- uploadObject();
+ store.uploadObject(key, tmpFile);
} else {
- multipartUploadObject();
+ store.multipartUploadObject(key, tmpFile);
}
} finally {
if (!tmpFile.delete()) {
@@ -124,91 +95,7 @@ public class AliyunOSSOutputStream extends OutputStream {
}
}
- /**
- * Upload temporary file as an OSS object, using single upload.
- *
- * @throws IOException
- */
- private void uploadObject() throws IOException {
- File object = tmpFile.getAbsoluteFile();
- FileInputStream fis = new FileInputStream(object);
- ObjectMetadata meta = new ObjectMetadata();
- meta.setContentLength(object.length());
- if (!serverSideEncryptionAlgorithm.isEmpty()) {
- meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
- }
- try {
- PutObjectResult result = ossClient.putObject(bucketName, key, fis, meta);
- LOG.debug(result.getETag());
- statistics.incrementWriteOps(1);
- } finally {
- fis.close();
- }
- }
-
- /**
- * Upload temporary file as an OSS object, using multipart upload.
- *
- * @throws IOException
- */
- private void multipartUploadObject() throws IOException {
- File object = tmpFile.getAbsoluteFile();
- long dataLen = object.length();
- long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize);
- int partNum = (int)(dataLen / realPartSize);
- if (dataLen % realPartSize != 0) {
- partNum += 1;
- }
-
- InitiateMultipartUploadRequest initiateMultipartUploadRequest =
- new InitiateMultipartUploadRequest(bucketName, key);
- ObjectMetadata meta = new ObjectMetadata();
- // meta.setContentLength(dataLen);
- if (!serverSideEncryptionAlgorithm.isEmpty()) {
- meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
- }
- initiateMultipartUploadRequest.setObjectMetadata(meta);
- InitiateMultipartUploadResult initiateMultipartUploadResult =
- ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
- List<PartETag> partETags = new ArrayList<PartETag>();
- String uploadId = initiateMultipartUploadResult.getUploadId();
- try {
- for (int i = 0; i < partNum; i++) {
- // TODO: Optimize this, avoid opening the object multiple times
- FileInputStream fis = new FileInputStream(object);
- try {
- long skipBytes = realPartSize * i;
- AliyunOSSUtils.skipFully(fis, skipBytes);
- long size = (realPartSize < dataLen - skipBytes) ?
- realPartSize : dataLen - skipBytes;
- UploadPartRequest uploadPartRequest = new UploadPartRequest();
- uploadPartRequest.setBucketName(bucketName);
- uploadPartRequest.setKey(key);
- uploadPartRequest.setUploadId(uploadId);
- uploadPartRequest.setInputStream(fis);
- uploadPartRequest.setPartSize(size);
- uploadPartRequest.setPartNumber(i + 1);
- UploadPartResult uploadPartResult =
- ossClient.uploadPart(uploadPartRequest);
- statistics.incrementWriteOps(1);
- partETags.add(uploadPartResult.getPartETag());
- } finally {
- fis.close();
- }
- }
- CompleteMultipartUploadRequest completeMultipartUploadRequest =
- new CompleteMultipartUploadRequest(bucketName, key,
- uploadId, partETags);
- CompleteMultipartUploadResult completeMultipartUploadResult =
- ossClient.completeMultipartUpload(completeMultipartUploadRequest);
- LOG.debug(completeMultipartUploadResult.getETag());
- } catch (OSSException | ClientException e) {
- AbortMultipartUploadRequest abortMultipartUploadRequest =
- new AbortMultipartUploadRequest(bucketName, key, uploadId);
- ossClient.abortMultipartUpload(abortMultipartUploadRequest);
- }
- }
@Override
public synchronized void flush() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
index d54dd9c..b96aea7 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
@@ -20,142 +20,58 @@ package org.apache.hadoop.fs.aliyun.oss;
import java.io.IOException;
import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
import java.net.URI;
-import java.net.URLDecoder;
-import java.util.Objects;
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.aliyun.oss.common.auth.DefaultCredentialProvider;
+import com.aliyun.oss.common.auth.DefaultCredentials;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ProviderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.fs.aliyun.oss.Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT;
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+import static org.apache.hadoop.fs.aliyun.oss.Constants.ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY;
/**
* Utility methods for Aliyun OSS code.
*/
final public class AliyunOSSUtils {
- private AliyunOSSUtils() {
- }
-
- /**
- * User information includes user name and password.
- */
- static public class UserInfo {
- private final String user;
- private final String password;
-
- public static final UserInfo EMPTY = new UserInfo("", "");
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AliyunOSSUtils.class);
- public UserInfo(String user, String password) {
- this.user = user;
- this.password = password;
- }
-
- /**
- * Predicate to verify user information is set.
- * @return true if the username is defined (not null, not empty).
- */
- public boolean hasLogin() {
- return StringUtils.isNotEmpty(user);
- }
-
- /**
- * Equality test matches user and password.
- * @param o other object
- * @return true if the objects are considered equivalent.
- */
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- UserInfo that = (UserInfo) o;
- return Objects.equals(user, that.user) &&
- Objects.equals(password, that.password);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(user, password);
- }
-
- public String getUser() {
- return user;
- }
-
- public String getPassword() {
- return password;
- }
+ private AliyunOSSUtils() {
}
/**
- * Used to get password from configuration, if default value is not available.
+ * Used to get password from configuration.
+ *
* @param conf configuration that contains password information
* @param key the key of the password
- * @param val the default value of the key
* @return the value for the key
* @throws IOException if failed to get password from configuration
*/
- static public String getPassword(Configuration conf, String key, String val)
+ static public String getPassword(Configuration conf, String key)
throws IOException {
- if (StringUtils.isEmpty(val)) {
- try {
- final char[] pass = conf.getPassword(key);
- if (pass != null) {
- return (new String(pass)).trim();
- } else {
- return "";
- }
- } catch (IOException ioe) {
- throw new IOException("Cannot find password option " + key, ioe);
- }
- } else {
- return val;
- }
- }
-
- /**
- * Extract the user information details from a URI.
- * @param name URI of the filesystem.
- * @return a login tuple, possibly empty.
- */
- public static UserInfo extractLoginDetails(URI name) {
try {
- String authority = name.getAuthority();
- if (authority == null) {
- return UserInfo.EMPTY;
- }
- int loginIndex = authority.indexOf('@');
- if (loginIndex < 0) {
- // No user information
- return UserInfo.EMPTY;
- }
- String login = authority.substring(0, loginIndex);
- int loginSplit = login.indexOf(':');
- if (loginSplit > 0) {
- String user = login.substring(0, loginSplit);
- String password = URLDecoder.decode(login.substring(loginSplit + 1),
- "UTF-8");
- return new UserInfo(user, password);
- } else if (loginSplit == 0) {
- // There is no user, just a password.
- return UserInfo.EMPTY;
+ final char[] pass = conf.getPassword(key);
+ if (pass != null) {
+ return (new String(pass)).trim();
} else {
- return new UserInfo(login, "");
+ return "";
}
- } catch (UnsupportedEncodingException e) {
- // This should never happen; translate it if it does.
- throw new RuntimeException(e);
+ } catch (IOException ioe) {
+ throw new IOException("Cannot find password option " + key, ioe);
}
}
/**
- * Skips the requested number of bytes or fail if there are not enough left.
- * This allows for the possibility that {@link InputStream#skip(long)} may not
- * skip as many bytes as requested (most likely because of reaching EOF).
+ * Skip the requested number of bytes or fail if there are no enough bytes
+ * left. This allows for the possibility that {@link InputStream#skip(long)}
+ * may not skip as many bytes as requested (most likely because of reaching
+ * EOF).
+ *
* @param is the input stream to skip.
* @param n the number of bytes to skip.
* @throws IOException thrown when skipped less number of bytes.
@@ -179,12 +95,69 @@ final public class AliyunOSSUtils {
* Calculate a proper size of multipart piece. If <code>minPartSize</code>
* is too small, the number of multipart pieces may exceed the limit of
* {@link Constants#MULTIPART_UPLOAD_PART_NUM_LIMIT}.
+ *
* @param contentLength the size of file.
* @param minPartSize the minimum size of multipart piece.
* @return a revisional size of multipart piece.
- */
+ */
public static long calculatePartSize(long contentLength, long minPartSize) {
long tmpPartSize = contentLength / MULTIPART_UPLOAD_PART_NUM_LIMIT + 1;
return Math.max(minPartSize, tmpPartSize);
}
+
+ /**
+ * Create credential provider specified by configuration, or create default
+ * credential provider if not specified.
+ *
+ * @param name the uri of the file system
+ * @param conf configuration
+ * @return a credential provider
+ * @throws IOException on any problem. Class construction issues may be
+ * nested inside the IOE.
+ */
+ public static CredentialsProvider getCredentialsProvider(URI name,
+ Configuration conf) throws IOException {
+ URI uri = java.net.URI.create(
+ name.getScheme() + "://" + name.getAuthority());
+ CredentialsProvider credentials;
+
+ String className = conf.getTrimmed(ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY);
+ if (StringUtils.isEmpty(className)) {
+ Configuration newConf =
+ ProviderUtils.excludeIncompatibleCredentialProviders(conf,
+ AliyunOSSFileSystem.class);
+ String accessKey =
+ AliyunOSSUtils.getPassword(newConf, ACCESS_KEY);
+ String secretKey =
+ AliyunOSSUtils.getPassword(newConf, SECRET_KEY);
+ credentials = new DefaultCredentialProvider(
+ new DefaultCredentials(accessKey, secretKey));
+ } else {
+ try {
+ LOG.debug("Credential provider class is:" + className);
+ Class<?> credClass = Class.forName(className);
+ try {
+ credentials =
+ (CredentialsProvider)credClass.getDeclaredConstructor(
+ URI.class, Configuration.class).newInstance(uri, conf);
+ } catch (NoSuchMethodException | SecurityException e) {
+ credentials =
+ (CredentialsProvider)credClass.getDeclaredConstructor()
+ .newInstance();
+ }
+ } catch (ClassNotFoundException e) {
+ throw new IOException(className + " not found.", e);
+ } catch (NoSuchMethodException | SecurityException e) {
+ throw new IOException(String.format("%s constructor exception. A " +
+ "class specified in %s must provide an accessible constructor " +
+ "accepting URI and Configuration, or an accessible default " +
+ "constructor.", className, ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY),
+ e);
+ } catch (ReflectiveOperationException | IllegalArgumentException e) {
+ throw new IOException(className + " instantiation exception.", e);
+ }
+ }
+
+ return credentials;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
index 9902275..243fdd4 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -72,7 +72,7 @@ public final class Constants {
// Number of records to get while paging through a directory listing
public static final String MAX_PAGING_KEYS_KEY = "fs.oss.paging.maximum";
- public static final int MAX_PAGING_KEYS_DEFAULT = 500;
+ public static final int MAX_PAGING_KEYS_DEFAULT = 1000;
// Size of each of or multipart pieces in bytes
public static final String MULTIPART_UPLOAD_SIZE_KEY =
@@ -109,5 +109,6 @@ public final class Constants {
public static final String FS_OSS = "oss";
public static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 100 * 1024L;
+ public static final int MAX_RETRIES = 10;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java
index de4e5a9..f234d50 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java
@@ -73,11 +73,6 @@ public class TestOSSFileSystemContract extends FileSystemContractBaseTest {
// not supported
}
- /**
- * Assert that root directory renames are not allowed.
- *
- * @throws Exception on failures
- */
@Override
public void testRootDirAlwaysExists() throws Exception {
//this will throw an exception if the path is not found
@@ -88,11 +83,6 @@ public class TestOSSFileSystemContract extends FileSystemContractBaseTest {
fs.exists(super.path("/")));
}
- /**
- * Assert that root directory renames are not allowed.
- *
- * @throws Exception on failures
- */
@Override
public void testRenameRootDirForbidden() throws Exception {
if (!renameSupported()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java
new file mode 100644
index 0000000..6331ed8
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.*;
+import java.net.URI;
+import java.security.DigestInputStream;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeNotNull;
+
+/**
+ * Test the bridging logic between Hadoop's abstract filesystem and
+ * Aliyun OSS.
+ */
+public class TestOSSFileSystemStore {
+ private Configuration conf;
+ private AliyunOSSFileSystemStore store;
+ private AliyunOSSFileSystem fs;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ fs = new AliyunOSSFileSystem();
+ fs.initialize(URI.create(conf.get("test.fs.oss.name")), conf);
+ store = fs.getStore();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try {
+ store.purge("test");
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @BeforeClass
+ public static void checkSettings() throws Exception {
+ Configuration conf = new Configuration();
+ assumeNotNull(conf.get("fs.oss.accessKeyId"));
+ assumeNotNull(conf.get("fs.oss.accessKeySecret"));
+ assumeNotNull(conf.get("test.fs.oss.name"));
+ }
+
+ protected void writeRenameReadCompare(Path path, long len)
+ throws IOException, NoSuchAlgorithmException {
+ // If len > fs.oss.multipart.upload.threshold,
+ // we'll use a multipart upload copy
+ MessageDigest digest = MessageDigest.getInstance("MD5");
+ OutputStream out = new BufferedOutputStream(
+ new DigestOutputStream(fs.create(path, false), digest));
+ for (long i = 0; i < len; i++) {
+ out.write('Q');
+ }
+ out.flush();
+ out.close();
+
+ assertTrue("Exists", fs.exists(path));
+
+ Path copyPath = path.suffix(".copy");
+ fs.rename(path, copyPath);
+
+ assertTrue("Copy exists", fs.exists(copyPath));
+
+ // Download file from Aliyun OSS and compare the digest against the original
+ MessageDigest digest2 = MessageDigest.getInstance("MD5");
+ InputStream in = new BufferedInputStream(
+ new DigestInputStream(fs.open(copyPath), digest2));
+ long copyLen = 0;
+ while (in.read() != -1) {
+ copyLen++;
+ }
+ in.close();
+
+ assertEquals("Copy length matches original", len, copyLen);
+ assertArrayEquals("Digests match", digest.digest(), digest2.digest());
+ }
+
+ @Test
+ public void testSmallUpload() throws IOException, NoSuchAlgorithmException {
+ // Regular upload, regular copy
+ writeRenameReadCompare(new Path("/test/small"), 16384);
+ }
+
+ @Test
+ public void testLargeUpload()
+ throws IOException, NoSuchAlgorithmException {
+ // Multipart upload, multipart copy
+ writeRenameReadCompare(new Path("/test/xlarge"), 52428800L); // 50MB byte
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java
index 8214b9f..f90a8bb 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.fs.aliyun.oss.contract;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.aliyun.oss.OSSTestUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java
new file mode 100644
index 0000000..eb0c5e0
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+/**
+ * Contract test suite covering Aliyun OSS integration with DistCp.
+ */
+public class TestOSSContractDispCp extends AbstractContractDistCpTest {
+
+ private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration newConf = super.createConfiguration();
+ newConf.setLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, MULTIPART_SETTING);
+ newConf.setLong(MULTIPART_UPLOAD_SIZE_KEY, MULTIPART_SETTING);
+ return newConf;
+ }
+
+ @Override
+ protected OSSContract createContract(Configuration conf) {
+ return new OSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java
new file mode 100644
index 0000000..cc21a2e
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test getFileStatus and related listing operations.
+ */
+public class TestOSSContractGetFileStatus
+ extends AbstractContractGetFileStatusTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OSSContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33e928f/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java
new file mode 100644
index 0000000..cbc262c
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Root dir operations against an Aliyun OSS bucket.
+ */
+public class TestOSSContractRootDir extends
+ AbstractContractRootDirectoryTest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestOSSContractRootDir.class);
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OSSContract(conf);
+ }
+
+ @Override
+ public void testListEmptyRootDirectory() throws IOException {
+ for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) {
+ try {
+ super.testListEmptyRootDirectory();
+ break;
+ } catch (AssertionError | FileNotFoundException e) {
+ if (attempt < maxAttempts) {
+ LOG.info("Attempt {} of {} for empty root directory test failed. "
+ + "Attempting retry.", attempt, maxAttempts);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e2) {
+ Thread.currentThread().interrupt();
+ fail("Test interrupted.");
+ break;
+ }
+ } else {
+ LOG.error(
+ "Empty root directory test failed {} attempts. Failing test.",
+ maxAttempts);
+ throw e;
+ }
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/22] hadoop git commit: Add CHANGES and release notes for
3.0.0-alpha1 to site
Posted by sh...@apache.org.
Add CHANGES and release notes for 3.0.0-alpha1 to site
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f388a209
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f388a209
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f388a209
Branch: refs/heads/HADOOP-12756
Commit: f388a209431c51e4062cc4a947274220870558a7
Parents: d152557
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Sep 6 09:51:48 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Sep 6 09:51:55 2016 -0700
----------------------------------------------------------------------
.../3.0.0-alpha1/CHANGES.3.0.0-alpha1.md | 3712 ++++++++++++++++++
.../3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md | 1854 +++++++++
2 files changed, 5566 insertions(+)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/22] hadoop git commit: HADOOP-13499. Support session credentials
for authenticating with Aliyun. Contributed by Genmao Yu.
Posted by sh...@apache.org.
HADOOP-13499. Support session credentials for authenticating with Aliyun. Contributed by Genmao Yu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bb741b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bb741b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bb741b9
Branch: refs/heads/HADOOP-12756
Commit: 6bb741b9f811d3a1c0ce4ecc91a78ac47513bb8e
Parents: bd2d97a
Author: Mingfei <mi...@intel.com>
Authored: Tue Aug 23 17:10:00 2016 +0800
Committer: Mingfei <mi...@intel.com>
Committed: Wed Sep 7 11:15:48 2016 +0800
----------------------------------------------------------------------
.../apache/hadoop/fs/aliyun/oss/Constants.java | 5 +-
.../oss/TemporaryAliyunCredentialsProvider.java | 64 ++++++++++++++++++++
.../aliyun/oss/TestOSSTemporaryCredentials.java | 64 ++++++++++++++++++++
3 files changed, 131 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb741b9/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
index 4ee4cd4..0bc6d57 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -31,8 +31,9 @@ public final class Constants {
"fs.oss.credentials.provider";
// OSS access verification
- public static final String ACCESS_KEY = "fs.oss.access.key";
- public static final String SECRET_KEY = "fs.oss.secret.key";
+ public static final String ACCESS_KEY = "fs.oss.accessKeyId";
+ public static final String SECRET_KEY = "fs.oss.accessKeySecret";
+ public static final String SECURITY_TOKEN = "fs.oss.securityToken";
// Number of simultaneous connections to oss
public static final String MAXIMUM_CONNECTIONS_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb741b9/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/TemporaryAliyunCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/TemporaryAliyunCredentialsProvider.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/TemporaryAliyunCredentialsProvider.java
new file mode 100644
index 0000000..ec8e7fe
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/TemporaryAliyunCredentialsProvider.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import com.aliyun.oss.common.auth.Credentials;
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.aliyun.oss.common.auth.DefaultCredentials;
+import com.aliyun.oss.common.auth.InvalidCredentialsException;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import java.net.URI;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+/**
+ * Support session credentials for authenticating with ALiyun.
+ */
+public class TemporaryAliyunCredentialsProvider implements CredentialsProvider {
+ public static final String NAME
+ = "org.apache.hadoop.fs.aliyun.oss.TemporaryAliyunCredentialsProvider";
+ private final String accessKeyId;
+ private final String accessKeySecret;
+ private final String securityToken;
+
+ public TemporaryAliyunCredentialsProvider(URI uri, Configuration conf) {
+ this.accessKeyId = conf.get(ACCESS_KEY, null);
+ this.accessKeySecret = conf.get(SECRET_KEY, null);
+ this.securityToken = conf.get(SECURITY_TOKEN, null);
+ }
+
+ @Override
+ public void setCredentials(Credentials creds) {
+
+ }
+
+ @Override
+ public Credentials getCredentials() {
+ if (!StringUtils.isEmpty(accessKeyId)
+ && !StringUtils.isEmpty(accessKeySecret)
+ && !StringUtils.isEmpty(securityToken)) {
+ return new DefaultCredentials(accessKeyId, accessKeySecret,
+ securityToken);
+ }
+ throw new InvalidCredentialsException(
+ "AccessKeyId, AccessKeySecret or SecurityToken is unset");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bb741b9/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java
new file mode 100644
index 0000000..ca2c0bc
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import com.aliyun.oss.common.auth.Credentials;
+import com.aliyun.oss.common.auth.InvalidCredentialsException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.aliyun.oss.contract.OSSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.junit.Test;
+
+import java.net.URI;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY;
+import static org.apache.hadoop.fs.aliyun.oss.Constants.SECRET_KEY;
+import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN;
+
+/**
+ * Tests use of temporary credentials (for example, Aliyun STS & Aliyun OSS).
+ * This test extends a class that "does things to the root directory", and
+ * should only be used against transient filesystems where you don't care about
+ * the data.
+ */
+public class TestOSSTemporaryCredentials extends AbstractFSContractTestBase {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OSSContract(conf);
+ }
+
+ @Test
+ public void testTemporaryCredentialValidation() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.set(ACCESS_KEY, "accessKeyId");
+ conf.set(SECRET_KEY, "accessKeySecret");
+ conf.set(SECURITY_TOKEN, "");
+ URI uri = getFileSystem().getUri();
+ TemporaryAliyunCredentialsProvider provider
+ = new TemporaryAliyunCredentialsProvider(uri, conf);
+ try {
+ Credentials credentials = provider.getCredentials();
+ fail("Expected a CredentialInitializationException, got " + credentials);
+ } catch (InvalidCredentialsException expected) {
+ // expected
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/22] hadoop git commit: HADOOP-13491. Fix several warnings from
findbugs. Contributed by Genmao Yu.
Posted by sh...@apache.org.
HADOOP-13491. Fix several warnings from findbugs. Contributed by Genmao Yu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d84c814
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d84c814
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d84c814
Branch: refs/heads/HADOOP-12756
Commit: 4d84c814fcaf074022593c057d8f8dec4cd461fa
Parents: a5d5342
Author: Mingfei <mi...@intel.com>
Authored: Thu Aug 18 16:06:57 2016 +0800
Committer: Mingfei <mi...@intel.com>
Committed: Wed Sep 7 11:15:47 2016 +0800
----------------------------------------------------------------------
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 3 ++-
.../fs/aliyun/oss/AliyunOSSInputStream.java | 6 ++---
.../fs/aliyun/oss/AliyunOSSOutputStream.java | 10 +++++---
.../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 26 +++++++++++++++++++-
4 files changed, 37 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d84c814/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 30ddf8c..6923b95 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.aliyun.oss;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+import com.aliyun.oss.ClientException;
import com.aliyun.oss.common.auth.CredentialsProvider;
import com.aliyun.oss.common.auth.DefaultCredentialProvider;
import com.aliyun.oss.common.auth.DefaultCredentials;
@@ -782,7 +783,7 @@ public class AliyunOSSFileSystem extends FileSystem {
ossClient.completeMultipartUpload(completeMultipartUploadRequest);
LOG.debug(completeMultipartUploadResult.getETag());
return true;
- } catch (Exception e) {
+ } catch (OSSException | ClientException e) {
AbortMultipartUploadRequest abortMultipartUploadRequest =
new AbortMultipartUploadRequest(bucketName, dstKey, uploadId);
ossClient.abortMultipartUpload(abortMultipartUploadRequest);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d84c814/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
index bcd00dc..b12e3f0 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
@@ -227,12 +227,12 @@ public class AliyunOSSInputStream extends FSInputStream {
}
@Override
- public void seek(long pos) throws IOException {
+ public synchronized void seek(long pos) throws IOException {
checkNotClosed();
if (position == pos) {
return;
} else if (pos > position && pos < position + partRemaining) {
- wrappedStream.skip(pos - position);
+ AliyunOSSUtils.skipFully(wrappedStream, pos - position);
position = pos;
} else {
reopen(pos);
@@ -240,7 +240,7 @@ public class AliyunOSSInputStream extends FSInputStream {
}
@Override
- public long getPos() throws IOException {
+ public synchronized long getPos() throws IOException {
checkNotClosed();
return position;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d84c814/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
index 589e014..654b81d 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
@@ -29,6 +29,8 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
+import com.aliyun.oss.ClientException;
+import com.aliyun.oss.OSSException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -113,7 +115,9 @@ public class AliyunOSSOutputStream extends OutputStream {
multipartUploadObject();
}
} finally {
- tmpFile.delete();
+ if (!tmpFile.delete()) {
+ LOG.warn("Can not delete file: " + tmpFile);
+ }
}
}
@@ -174,7 +178,7 @@ public class AliyunOSSOutputStream extends OutputStream {
FileInputStream fis = new FileInputStream(object);
try {
long skipBytes = partSize * i;
- fis.skip(skipBytes);
+ AliyunOSSUtils.skipFully(fis, skipBytes);
long size = (partSize < dataLen - skipBytes) ?
partSize : dataLen - skipBytes;
UploadPartRequest uploadPartRequest = new UploadPartRequest();
@@ -198,7 +202,7 @@ public class AliyunOSSOutputStream extends OutputStream {
CompleteMultipartUploadResult completeMultipartUploadResult =
ossClient.completeMultipartUpload(completeMultipartUploadRequest);
LOG.debug(completeMultipartUploadResult.getETag());
- } catch (Exception e) {
+ } catch (OSSException | ClientException e) {
AbortMultipartUploadRequest abortMultipartUploadRequest =
new AbortMultipartUploadRequest(bucketName, key, uploadId);
ossClient.abortMultipartUpload(abortMultipartUploadRequest);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d84c814/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
index 3f66a4f..9acde00 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.aliyun.oss;
import java.io.IOException;
+import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URLDecoder;
@@ -116,7 +117,7 @@ final public class AliyunOSSUtils {
/**
* Extract the user information details from a URI.
- * @param name URI of the filesystem
+ * @param name URI of the filesystem.
* @return a login tuple, possibly empty.
*/
public static UserInfo extractLoginDetails(URI name) {
@@ -148,4 +149,27 @@ final public class AliyunOSSUtils {
throw new RuntimeException(e);
}
}
+
+ /**
+ * Skips the requested number of bytes or fail if there are not enough left.
+ * This allows for the possibility that {@link InputStream#skip(long)} may not
+ * skip as many bytes as requested (most likely because of reaching EOF).
+ * @param is the input stream to skip.
+ * @param n the number of bytes to skip.
+ * @throws IOException thrown when skipped less number of bytes.
+ */
+ public static void skipFully(InputStream is, long n) throws IOException {
+ long total = 0;
+ long cur = 0;
+
+ do {
+ cur = is.skip(n - total);
+ total += cur;
+ } while((total < n) && (cur > 0));
+
+ if (total < n) {
+ throw new IOException("Failed to skip " + n + " bytes, possibly due " +
+ "to EOF.");
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/22] hadoop git commit: Add CHANGES and release notes for
3.0.0-alpha1 to site
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f388a209/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md
new file mode 100644
index 0000000..edf47ca
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md
@@ -0,0 +1,1854 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# "Apache Hadoop" 3.0.0-alpha1 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [HDFS-46](https://issues.apache.org/jira/browse/HDFS-46) | *Major* | **The namespace quota of root directory should not be Integer.MAX\_VALUE**
+
+Change default namespace quota of root directory from Integer.MAX\_VALUE to Long.MAX\_VALUE.
+
+
+---
+
+* [HADOOP-8124](https://issues.apache.org/jira/browse/HADOOP-8124) | *Major* | **Remove the deprecated Syncable.sync() method**
+
+Remove the deprecated FSDataOutputStream constructor, FSDataOutputStream.sync() and Syncable.sync().
+
+
+---
+
+* [HDFS-3034](https://issues.apache.org/jira/browse/HDFS-3034) | *Major* | **Remove the deprecated Syncable.sync() method**
+
+Remove the deprecated DFSOutputStream.sync() method.
+
+
+---
+
+* [HADOOP-7659](https://issues.apache.org/jira/browse/HADOOP-7659) | *Minor* | **fs -getmerge isn't guaranteed to work well over non-HDFS filesystems**
+
+Documented that the "fs -getmerge" shell command may not work properly over non HDFS-filesystem implementations due to platform-varying file list ordering.
+
+
+---
+
+* [HADOOP-8776](https://issues.apache.org/jira/browse/HADOOP-8776) | *Minor* | **Provide an option in test-patch that can enable / disable compiling native code**
+
+test-patch.sh adds a new option "--build-native". When set to false native
+components are not built. When set to true native components are built. The
+default value is true.
+
+
+---
+
+* [HDFS-5079](https://issues.apache.org/jira/browse/HDFS-5079) | *Major* | **Cleaning up NNHAStatusHeartbeat.State DatanodeProtocolProtos.**
+
+This change affects wire-compatibility of the NameNode/DataNode heartbeat protocol.
+
+
+---
+
+* [HDFS-5570](https://issues.apache.org/jira/browse/HDFS-5570) | *Major* | **Deprecate hftp / hsftp and replace them with webhdfs / swebhdfs**
+
+Support for hftp and hsftp has been removed. They have superseded by webhdfs and swebhdfs.
+
+
+---
+
+* [HADOOP-10485](https://issues.apache.org/jira/browse/HADOOP-10485) | *Major* | **Remove dead classes in hadoop-streaming**
+
+Deprecated and unused classes in the org.apache.hadoop.record package have been removed from hadoop-streaming.
+
+
+---
+
+* [HDFS-6246](https://issues.apache.org/jira/browse/HDFS-6246) | *Minor* | **Remove 'dfs.support.append' flag from trunk code**
+
+Appends in HDFS can no longer be disabled.
+
+
+---
+
+* [HADOOP-10474](https://issues.apache.org/jira/browse/HADOOP-10474) | *Major* | **Move o.a.h.record to hadoop-streaming**
+
+The classes in org.apache.hadoop.record are moved from hadoop-common to a new hadoop-streaming artifact within the hadoop-tools module.
+
+
+---
+
+* [HDFS-2538](https://issues.apache.org/jira/browse/HDFS-2538) | *Minor* | **option to disable fsck dots**
+
+fsck does not print out dots for progress reporting by default. To print out dots, you should specify '-showprogress' option.
+
+
+---
+
+* [HADOOP-9902](https://issues.apache.org/jira/browse/HADOOP-9902) | *Major* | **Shell script rewrite**
+
+<!-- markdown -->
+The Hadoop shell scripts have been rewritten to fix many long standing bugs and include some new features. While an eye has been kept towards compatibility, some changes may break existing installations.
+
+INCOMPATIBLE CHANGES:
+
+* The pid and out files for secure daemons have been renamed to include the appropriate ${HADOOP\_IDENT\_STR}. This should allow, with proper configurations in place, for multiple versions of the same secure daemon to run on a host. Additionally, pid files are now created when daemons are run in interactive mode. This will also prevent the accidental starting of two daemons with the same configuration prior to launching java (i.e., "fast fail" without having to wait for socket opening).
+* All Hadoop shell script subsystems now execute hadoop-env.sh, which allows for all of the environment variables to be in one location. This was not the case previously.
+* The default content of *-env.sh has been significantly altered, with the majority of defaults moved into more protected areas inside the code. Additionally, these files do not auto-append anymore; setting a variable on the command line prior to calling a shell command must contain the entire content, not just any extra settings. This brings Hadoop more in-line with the vast majority of other software packages.
+* All HDFS\_\*, YARN\_\*, and MAPRED\_\* environment variables act as overrides to their equivalent HADOOP\_\* environment variables when 'hdfs', 'yarn', 'mapred', and related commands are executed. Previously, these were separated out which meant a significant amount of duplication of common settings.
+* hdfs-config.sh and hdfs-config.cmd were inadvertently duplicated into libexec and sbin. The sbin versions have been removed.
+* The log4j settings forcibly set by some *-daemon.sh commands have been removed. These settings are now configurable in the \*-env.sh files via \*\_OPT.
+* Support for various undocumented YARN log4j.properties files has been removed.
+* Support for ${HADOOP\_MASTER} and the related rsync code have been removed.
+* The undocumented and unused yarn.id.str Java property has been removed.
+* The unused yarn.policy.file Java property has been removed.
+* We now require bash v3 (released July 27, 2004) or better in order to take advantage of better regex handling and ${BASH\_SOURCE}. POSIX sh will not work.
+* Support for --script has been removed. We now use ${HADOOP\_\*\_PATH} or ${HADOOP\_PREFIX} to find the necessary binaries. (See other note regarding ${HADOOP\_PREFIX} auto discovery.)
+* Non-existent classpaths, ld.so library paths, JNI library paths, etc, will be ignored and stripped from their respective environment settings.
+
+NEW FEATURES:
+
+* Daemonization has been moved from *-daemon.sh to the bin commands via the --daemon option. Simply use --daemon start to start a daemon, --daemon stop to stop a daemon, and --daemon status to set $? to the daemon's status. The return code for status is LSB-compatible. For example, 'hdfs --daemon start namenode'.
+* It is now possible to override some of the shell code capabilities to provide site specific functionality without replacing the shipped versions. Replacement functions should go into the new hadoop-user-functions.sh file.
+* A new option called --buildpaths will attempt to add developer build directories to the classpath to allow for in source tree testing.
+* Operations which trigger ssh connections can now use pdsh if installed. ${HADOOP\_SSH\_OPTS} still gets applied.
+* Added distch and jnipath subcommands to the hadoop command.
+* Shell scripts now support a --debug option which will report basic information on the construction of various environment variables, java options, classpath, etc. to help in configuration debugging.
+
+BUG FIXES:
+
+* ${HADOOP\_CONF\_DIR} is now properly honored everywhere, without requiring symlinking and other such tricks.
+* ${HADOOP\_CONF\_DIR}/hadoop-layout.sh is now documented with a provided hadoop-layout.sh.example file.
+* Shell commands should now work properly when called as a relative path, without ${HADOOP\_PREFIX} being defined, and as the target of bash -x for debugging. If ${HADOOP\_PREFIX} is not set, it will be automatically determined based upon the current location of the shell library. Note that other parts of the extended Hadoop ecosystem may still require this environment variable to be configured.
+* Operations which trigger ssh will now limit the number of connections to run in parallel to ${HADOOP\_SSH\_PARALLEL} to prevent memory and network exhaustion. By default, this is set to 10.
+* ${HADOOP\_CLIENT\_OPTS} support has been added to a few more commands.
+* Some subcommands were not listed in the usage.
+* Various options on hadoop command lines were supported inconsistently. These have been unified into hadoop-config.sh. --config is still required to be first, however.
+* ulimit logging for secure daemons no longer assumes /bin/bash but does assume bash is on the command line path.
+* Removed references to some Yahoo! specific paths.
+* Removed unused slaves.sh from YARN build tree.
+* Many exit states have been changed to reflect reality.
+* Shell level errors now go to STDERR. Before, many of them went incorrectly to STDOUT.
+* CDPATH with a period (.) should no longer break the scripts.
+* The scripts no longer try to chown directories.
+* If ${JAVA\_HOME} is not set on OS X, it now properly detects it instead of throwing an error.
+
+IMPROVEMENTS:
+
+* The *.out files are now appended instead of overwritten to allow for external log rotation.
+* The style and layout of the scripts is much more consistent across subprojects.
+* More of the shell code is now commented.
+* Significant amounts of redundant code have been moved into a new file called hadoop-functions.sh.
+* The various *-env.sh have been massively changed to include documentation and examples on what can be set, ramifications of setting, etc. for all variables that are expected to be set by a user.
+* There is now some trivial de-duplication and sanitization of the classpath and JVM options. This allows, amongst other things, for custom settings in \*\_OPTS for Hadoop daemons to override defaults and other generic settings (i.e., ${HADOOP\_OPTS}). This is particularly relevant for Xmx settings, as one can now set them in _OPTS and ignore the heap specific options for daemons which force the size in megabytes.
+* Subcommands have been alphabetized in both usage and in the code.
+* All/most of the functionality provided by the sbin/* commands has been moved to either their bin/ equivalents or made into functions. The rewritten versions of these commands are now wrappers to maintain backward compatibility.
+* Usage information is given with the following options/subcommands for all scripts using the common framework: --? -? ? --help -help -h help
+* Several generic environment variables have been added to provide a common configuration for pids, logs, and their security equivalents. The older versions still act as overrides to these generic versions.
+* Groundwork has been laid to allow for custom secure daemon setup using something other than jsvc (e.g., pfexec on Solaris).
+* Scripts now test and report better error messages for various states of the log and pid dirs on daemon startup. Before, unprotected shell errors would be displayed to the user.
+
+
+---
+
+* [HADOOP-11041](https://issues.apache.org/jira/browse/HADOOP-11041) | *Minor* | **VersionInfo output specifies subversion**
+
+This changes the output of the 'hadoop version' command to generically say 'Source code repository' rather than specify which type of repo.
+
+
+---
+
+* [MAPREDUCE-5972](https://issues.apache.org/jira/browse/MAPREDUCE-5972) | *Trivial* | **Fix typo 'programatically' in job.xml (and a few other places)**
+
+Fix a typo. If a configuration is set through program, the source of the configuration is set to 'programmatically' instead of 'programatically' now.
+
+
+---
+
+* [MAPREDUCE-2841](https://issues.apache.org/jira/browse/MAPREDUCE-2841) | *Major* | **Task level native optimization**
+
+Adds a native implementation of the map output collector. The native library will build automatically with -Pnative. Users may choose the new collector on a job-by-job basis by setting mapreduce.job.map.output.collector.class=org.apache.hadoop.mapred.
+nativetask.NativeMapOutputCollectorDelegator in their job configuration. For shuffle-intensive jobs this may provide speed-ups of 30% or more.
+
+
+---
+
+* [HADOOP-11356](https://issues.apache.org/jira/browse/HADOOP-11356) | *Major* | **Removed deprecated o.a.h.fs.permission.AccessControlException**
+
+org.apache.hadoop.fs.permission.AccessControlException was deprecated in the last major release, and has been removed in favor of org.apache.hadoop.security.AccessControlException
+
+
+---
+
+* [HADOOP-10950](https://issues.apache.org/jira/browse/HADOOP-10950) | *Major* | **rework heap management vars**
+
+<!-- markdown -->
+
+* HADOOP\_HEAPSIZE variable has been deprecated (It will still be honored if set, but expect it to go away in the future). In its place, HADOOP\_HEAPSIZE\_MAX and HADOOP\_HEAPSIZE\_MIN have been introduced to set Xmx and Xms, respectively.
+
+* The internal variable JAVA\_HEAP\_MAX has been removed.
+
+* Default heap sizes have been removed. This will allow for the JVM to use auto-tuning based upon the memory size of the host. To re-enable the old default, configure HADOOP\_HEAPSIZE_MAX="1g" in hadoop-env.sh.
+
+* All global and daemon-specific heap size variables now support units. If the variable is only a number, the size is assumed to be in megabytes.
+
+
+---
+
+* [HADOOP-11353](https://issues.apache.org/jira/browse/HADOOP-11353) | *Major* | **Add support for .hadooprc**
+
+.hadooprc allows users a convenient way to set and/or override the shell level settings.
+
+
+---
+
+* [MAPREDUCE-5785](https://issues.apache.org/jira/browse/MAPREDUCE-5785) | *Major* | **Derive heap size or mapreduce.\*.memory.mb automatically**
+
+The memory values for mapreduce.map/reduce.memory.mb keys, if left to their default values of -1, will now be automatically inferred from the heap size value system property (-Xmx) specified for mapreduce.map/reduce.java.opts keys.
+
+The converse is also done, i.e. if mapreduce.map/reduce.memory.mb values are specified, but no -Xmx is supplied for mapreduce.map/reduce.java.opts keys, then the -Xmx value will be derived from the former's value.
+
+If neither is specified, then a default value of 1024 MB gets used.
+
+For both these conversions, a scaling factor specified by property mapreduce.job.heap.memory-mb.ratio is used, to account for overheads between heap usage vs. actual physical memory usage.
+
+Existing configs or job code that already specify both the set of properties explicitly would not be affected by this inferring change.
+
+
+---
+
+* [YARN-2428](https://issues.apache.org/jira/browse/YARN-2428) | *Trivial* | **LCE default banned user list should have yarn**
+
+The user 'yarn' is no longer allowed to run tasks for security reasons.
+
+
+---
+
+* [HADOOP-11460](https://issues.apache.org/jira/browse/HADOOP-11460) | *Major* | **Deprecate shell vars**
+
+<!-- markdown -->
+
+| Old | New |
+|:---- |:---- |
+| HADOOP\_HDFS\_LOG\_DIR | HADOOP\_LOG\_DIR |
+| HADOOP\_HDFS\_LOGFILE | HADOOP\_LOGFILE |
+| HADOOP\_HDFS\_NICENESS | HADOOP\_NICENESS |
+| HADOOP\_HDFS\_STOP\_TIMEOUT | HADOOP\_STOP\_TIMEOUT |
+| HADOOP\_HDFS\_PID\_DIR | HADOOP\_PID\_DIR |
+| HADOOP\_HDFS\_ROOT\_LOGGER | HADOOP\_ROOT\_LOGGER |
+| HADOOP\_HDFS\_IDENT\_STRING | HADOOP\_IDENT\_STRING |
+| HADOOP\_MAPRED\_LOG\_DIR | HADOOP\_LOG\_DIR |
+| HADOOP\_MAPRED\_LOGFILE | HADOOP\_LOGFILE |
+| HADOOP\_MAPRED\_NICENESS | HADOOP\_NICENESS |
+| HADOOP\_MAPRED\_STOP\_TIMEOUT | HADOOP\_STOP\_TIMEOUT |
+| HADOOP\_MAPRED\_PID\_DIR | HADOOP\_PID\_DIR |
+| HADOOP\_MAPRED\_ROOT\_LOGGER | HADOOP\_ROOT\_LOGGER |
+| HADOOP\_MAPRED\_IDENT\_STRING | HADOOP\_IDENT\_STRING |
+| YARN\_CONF\_DIR | HADOOP\_CONF\_DIR |
+| YARN\_LOG\_DIR | HADOOP\_LOG\_DIR |
+| YARN\_LOGFILE | HADOOP\_LOGFILE |
+| YARN\_NICENESS | HADOOP\_NICENESS |
+| YARN\_STOP\_TIMEOUT | HADOOP\_STOP\_TIMEOUT |
+| YARN\_PID\_DIR | HADOOP\_PID\_DIR |
+| YARN\_ROOT\_LOGGER | HADOOP\_ROOT\_LOGGER |
+| YARN\_IDENT\_STRING | HADOOP\_IDENT\_STRING |
+| YARN\_OPTS | HADOOP\_OPTS |
+| YARN\_SLAVES | HADOOP\_SLAVES |
+| YARN\_USER\_CLASSPATH | HADOOP\_CLASSPATH |
+| YARN\_USER\_CLASSPATH\_FIRST | HADOOP\_USER\_CLASSPATH\_FIRST |
+| KMS\_CONFIG | HADOOP\_CONF\_DIR |
+| KMS\_LOG | HADOOP\_LOG\_DIR |
+
+
+---
+
+* [HADOOP-7713](https://issues.apache.org/jira/browse/HADOOP-7713) | *Trivial* | **dfs -count -q should label output column**
+
+Added -v option to fs -count command to display a header record in the report.
+
+
+---
+
+* [HADOOP-11485](https://issues.apache.org/jira/browse/HADOOP-11485) | *Major* | **Pluggable shell integration**
+
+Support for shell profiles has been added. They allow for easy integration of additional functionality, classpaths, and more from inside the bash scripts rather than relying upon modifying hadoop-env.sh, etc. See the Unix Shell Guide for more information.
+
+
+---
+
+* [HADOOP-8934](https://issues.apache.org/jira/browse/HADOOP-8934) | *Minor* | **Shell command ls should include sort options**
+
+Options to sort output of fs -ls comment: -t (mtime), -S (size), -u (atime), -r (reverse)
+
+
+---
+
+* [HADOOP-11554](https://issues.apache.org/jira/browse/HADOOP-11554) | *Major* | **Expose HadoopKerberosName as a hadoop subcommand**
+
+The hadoop kerbname subcommand has been added to ease operational pain in determining the output of auth\_to\_local rules.
+
+
+---
+
+* [HDFS-7460](https://issues.apache.org/jira/browse/HDFS-7460) | *Major* | **Rewrite httpfs to use new shell framework**
+
+<!-- markdown -->
+This deprecates the following environment variables:
+
+| Old | New |
+|:---- |:---- |
+| HTTPFS_LOG | HADOOP_LOG_DIR|
+| HTTPFS_CONFG | HADOOP_CONF_DIR |
+
+
+---
+
+* [MAPREDUCE-5653](https://issues.apache.org/jira/browse/MAPREDUCE-5653) | *Major* | **DistCp does not honour config-overrides for mapreduce.[map,reduce].memory.mb**
+
+Prior to this change, distcp had hard-coded values for memory usage. Now distcp will honor memory settings in a way compatible with the rest of MapReduce.
+
+
+---
+
+* [HADOOP-11657](https://issues.apache.org/jira/browse/HADOOP-11657) | *Minor* | **Align the output of `hadoop fs -du` to be more Unix-like**
+
+The output of du has now been made more Unix-like, with aligned output.
+
+
+---
+
+* [HDFS-7302](https://issues.apache.org/jira/browse/HDFS-7302) | *Major* | **namenode -rollingUpgrade downgrade may finalize a rolling upgrade**
+
+Remove "downgrade" from "namenode -rollingUpgrade" startup option since it may incorrectly finalize an ongoing rolling upgrade.
+
+
+---
+
+* [HADOOP-6857](https://issues.apache.org/jira/browse/HADOOP-6857) | *Major* | **FsShell should report raw disk usage including replication factor**
+
+The output format of hadoop fs -du has been changed. It shows not only the file size but also the raw disk usage including the replication factor.
+
+
+---
+
+* [HADOOP-11226](https://issues.apache.org/jira/browse/HADOOP-11226) | *Major* | **Add a configuration to set ipc.Client's traffic class with IPTOS\_LOWDELAY\|IPTOS\_RELIABILITY**
+
+Use low latency TCP connections for hadoop IPC
+
+
+---
+
+* [HADOOP-10115](https://issues.apache.org/jira/browse/HADOOP-10115) | *Major* | **Exclude duplicate jars in hadoop package under different component's lib**
+
+Jars in the various subproject lib directories are now de-duplicated against Hadoop common. Users who interact directly with those directories must be sure to pull in common's dependencies as well.
+
+
+---
+
+* [YARN-3154](https://issues.apache.org/jira/browse/YARN-3154) | *Blocker* | **Should not upload partial logs for MR jobs or other "short-running' applications**
+
+Applications which made use of the LogAggregationContext in their application will need to revisit this code in order to make sure that their logs continue to get rolled out.
+
+
+---
+
+* [HADOOP-9477](https://issues.apache.org/jira/browse/HADOOP-9477) | *Major* | **Add posixGroups support for LDAP groups mapping service**
+
+Add posixGroups support for LDAP groups mapping service. The change in LDAPGroupMapping is compatible with previous scenario. In LDAP, the group mapping between {{posixAccount}} and {{posixGroup}} is different from the general LDAPGroupMapping, one of the differences is the {{"memberUid"}} will be used to mapping {{posixAccount}} and {{posixGroup}}. The feature will handle the mapping in internal when configuration {{hadoop.security.group.mapping.ldap.search.filter.user}} is set as "posixAccount" and {{hadoop.security.group.mapping.ldap.search.filter.group}} is "posixGroup".
+
+
+---
+
+* [MAPREDUCE-4424](https://issues.apache.org/jira/browse/MAPREDUCE-4424) | *Minor* | **'mapred job -list' command should show the job name as well**
+
+Now "mapred job -list" command displays the Job Name as well.
+
+
+---
+
+* [YARN-3241](https://issues.apache.org/jira/browse/YARN-3241) | *Major* | **FairScheduler handles "invalid" queue names inconsistently**
+
+FairScheduler does not allow queue names with leading or tailing spaces or empty sub-queue names anymore.
+
+
+---
+
+* [HDFS-7985](https://issues.apache.org/jira/browse/HDFS-7985) | *Major* | **WebHDFS should be always enabled**
+
+WebHDFS is mandatory and cannot be disabled.
+
+
+---
+
+* [HDFS-6353](https://issues.apache.org/jira/browse/HDFS-6353) | *Major* | **Check and make checkpoint before stopping the NameNode**
+
+Stopping the namenode on secure systems now requires the user be authenticated.
+
+
+---
+
+* [HADOOP-11553](https://issues.apache.org/jira/browse/HADOOP-11553) | *Blocker* | **Formalize the shell API**
+
+Python is now required to build the documentation.
+
+
+---
+
+* [HDFS-7501](https://issues.apache.org/jira/browse/HDFS-7501) | *Major* | **TransactionsSinceLastCheckpoint can be negative on SBNs**
+
+Fixed a bug where the StandbyNameNode's TransactionsSinceLastCheckpoint metric may slide into a negative number after every subsequent checkpoint.
+
+
+---
+
+* [HADOOP-11660](https://issues.apache.org/jira/browse/HADOOP-11660) | *Minor* | **Add support for hardware crc of HDFS checksums on ARM aarch64 architecture**
+
+Add support for aarch64 CRC instructions
+
+
+---
+
+* [HADOOP-11731](https://issues.apache.org/jira/browse/HADOOP-11731) | *Major* | **Rework the changelog and releasenotes**
+
+<!-- markdown -->
+* The release notes now only contains JIRA issues with incompatible changes and actual release notes. The generated format has been changed from HTML to markdown.
+
+* The changelog is now automatically generated from data stored in JIRA rather than manually maintained. The format has been changed from pure text to markdown as well as containing more of the information that was previously stored in the release notes.
+
+* In order to generate the changes file, python must be installed.
+
+* New -Preleasedocs profile added to maven in order to trigger this functionality.
+
+
+---
+
+* [YARN-3365](https://issues.apache.org/jira/browse/YARN-3365) | *Major* | **Add support for using the 'tc' tool via container-executor**
+
+Adding support for using the 'tc' tool in batch mode via container-executor. This is a prerequisite for traffic-shaping functionality that is necessary to support outbound bandwidth as a resource in YARN.
+
+
+---
+
+* [HADOOP-11781](https://issues.apache.org/jira/browse/HADOOP-11781) | *Major* | **fix race conditions and add URL support to smart-apply-patch.sh**
+
+Now auto-downloads patch from issue-id; fixed race conditions; fixed bug affecting some patches.
+
+
+---
+
+* [YARN-3443](https://issues.apache.org/jira/browse/YARN-3443) | *Major* | **Create a 'ResourceHandler' subsystem to ease addition of support for new resource types on the NM**
+
+The current cgroups implementation is closely tied to supporting CPU as a resource . This patch separates out CGroups implementation into a reusable class as well as provides a simple ResourceHandler subsystem that will enable us to add support for new resource types on the NM - e.g Network, Disk etc.
+
+
+---
+
+* [HDFS-6666](https://issues.apache.org/jira/browse/HDFS-6666) | *Minor* | **Abort NameNode and DataNode startup if security is enabled but block access token is not enabled.**
+
+NameNode and DataNode now abort during startup if attempting to run in secure mode, but block access tokens are not enabled by setting configuration property dfs.block.access.token.enable to true in hdfs-site.xml. Previously, this case logged a warning, because this would be an insecure configuration.
+
+
+---
+
+* [YARN-3021](https://issues.apache.org/jira/browse/YARN-3021) | *Major* | **YARN's delegation-token handling disallows certain trust setups to operate properly over DistCp**
+
+ResourceManager renews delegation tokens for applications. This behavior has been changed to renew tokens only if the token's renewer is a non-empty string. MapReduce jobs can instruct ResourceManager to skip renewal of tokens obtained from certain hosts by specifying the hosts with configuration mapreduce.job.hdfs-servers.token-renewal.exclude=\<host1\>,\<host2\>,..,\<hostN\>.
+
+
+---
+
+* [HADOOP-11746](https://issues.apache.org/jira/browse/HADOOP-11746) | *Major* | **rewrite test-patch.sh**
+
+<!-- markdown -->
+* test-patch.sh now has new output that is different than the previous versions
+* test-patch.sh is now pluggable via the test-patch.d directory, with checkstyle and shellcheck tests included
+* JIRA comments now use much more markup to improve readability
+* test-patch.sh now supports either a file name, a URL, or a JIRA issue as input in developer mode
+* If part of the patch testing code is changed, test-patch.sh will now attempt to re-executing itself using the new version.
+* Some logic to try and reduce the amount of unnecessary tests. For example, patches that only modify markdown should not run the Java compilation tests.
+* Plugins for checkstyle, shellcheck, and whitespace now execute as necessary.
+* New test code for mvn site
+* A breakdown of the times needed to execute certain blocks as well as a total runtime is now reported to assist in fixing long running tests and optimize the entire process.
+* Several new options
+ * --resetrepo will put test-patch.sh in destructive mode, similar to a normal Jenkins run
+ * --testlist allows one to provide a comma delimited list of test subsystems to forcibly execute
+ * --modulelist to provide a comma delimited list of module tests to execute in addition to the ones that are automatically detected
+ * --offline mode to attempt to stop connecting to the Internet for certain operations
+* test-patch.sh now defaults to the POSIX equivalents on Solaris and Illumos-based operating systems
+* shelldocs.py may be used to generate test-patch.sh API information
+* FindBugs output is now listed on the JIRA comment
+* lots of general code cleanup, including attempts to remove any local state files to reduce potential race conditions
+* Some logic to determine if a patch is for a given major branch using several strategies as well as a particular git ref (using git+ref as part of the name).
+* Some logic to determine if a patch references a particular JIRA issue.
+* Unit tests are only flagged as necessary with native or Java code, since Hadoop has no framework in place yet for other types of unit tests.
+* test-patch now exits with a failure status if problems arise trying to do git checkouts. Previously the exit code was success.
+
+
+---
+
+* [YARN-3366](https://issues.apache.org/jira/browse/YARN-3366) | *Major* | **Outbound network bandwidth : classify/shape traffic originating from YARN containers**
+
+1) A TrafficController class that provides an implementation for traffic shaping using tc.
+2) A ResourceHandler implementation for OutboundBandwidth as a resource - isolation/enforcement using cgroups and tc.
+
+
+---
+
+* [HADOOP-11861](https://issues.apache.org/jira/browse/HADOOP-11861) | *Major* | **test-patch.sh rewrite addendum patch**
+
+<!-- markdown -->
+* --build-native=false should work now
+* --branch option lets one specify a branch to test against on the command line
+* On certain Jenkins machines, the artifact directory sometimes gets deleted from outside the test-patch script. There is now some code to try to detect, alert, and quick exit if that happens.
+* Various semi-critical output and bug fixes
+
+
+---
+
+* [HADOOP-10597](https://issues.apache.org/jira/browse/HADOOP-10597) | *Major* | **RPC Server signals backoff to clients when all request queues are full**
+
+This change introduces a new configuration key used by RPC server to decide whether to send backoff signal to RPC Client when RPC call queue is full. When the feature is enabled, RPC server will no longer block on the processing of RPC requests when RPC call queue is full. It helps to improve quality of service when the service is under heavy load. The configuration key is in the format of "ipc.#port#.backoff.enable" where #port# is the port number that RPC server listens on. For example, if you want to enable the feature for the RPC server that listens on 8020, set ipc.8020.backoff.enable to true.
+
+
+---
+
+* [HADOOP-11627](https://issues.apache.org/jira/browse/HADOOP-11627) | *Major* | **Remove io.native.lib.available**
+
+io.native.lib.available was removed. Always use native libraries if they exist.
+
+
+---
+
+* [HADOOP-11843](https://issues.apache.org/jira/browse/HADOOP-11843) | *Major* | **Make setting up the build environment easier**
+
+Includes a docker based solution for setting up a build environment with minimal effort.
+
+
+---
+
+* [HDFS-7281](https://issues.apache.org/jira/browse/HDFS-7281) | *Major* | **Missing block is marked as corrupted block**
+
+The patch improves the reporting around missing blocks and corrupted blocks.
+
+1. A block is missing if and only if all DNs of its expected replicas are dead.
+2. A block is corrupted if and only if all its available replicas are corrupted. So if a block has 3 replicas; one of the DN is dead, the other two replicas are corrupted; it will be marked as corrupted.
+3. A new line is added to fsck output to display the corrupt block size per file.
+4. A new line is added to fsck output to display the number of missing blocks in the summary section.
+
+
+---
+
+* [HADOOP-11813](https://issues.apache.org/jira/browse/HADOOP-11813) | *Minor* | **releasedocmaker.py should use today's date instead of unreleased**
+
+Use today instead of 'Unreleased' in releasedocmaker.py when --usetoday is given as an option.
+
+
+---
+
+* [HDFS-8226](https://issues.apache.org/jira/browse/HDFS-8226) | *Blocker* | **Non-HA rollback compatibility broken**
+
+Non-HA rollback steps have been changed. Run the rollback command on the namenode (`bin/hdfs namenode -rollback`) before starting cluster with '-rollback' option using (sbin/start-dfs.sh -rollback).
+
+
+---
+
+* [MAPREDUCE-2632](https://issues.apache.org/jira/browse/MAPREDUCE-2632) | *Major* | **Avoid calling the partitioner when the numReduceTasks is 1.**
+
+A partitioner is now only created if there are multiple reducers.
+
+
+---
+
+* [HDFS-8241](https://issues.apache.org/jira/browse/HDFS-8241) | *Minor* | **Remove unused NameNode startup option -finalize**
+
+Remove -finalize option from hdfs namenode command.
+
+
+---
+
+* [HDFS-6888](https://issues.apache.org/jira/browse/HDFS-6888) | *Major* | **Allow selectively audit logging ops**
+
+Specific HDFS ops can be selectively excluded from audit logging via 'dfs.namenode.audit.log.debug.cmdlist' configuration.
+
+
+---
+
+* [HDFS-8157](https://issues.apache.org/jira/browse/HDFS-8157) | *Major* | **Writes to RAM DISK reserve locked memory for block files**
+
+This change requires setting the dfs.datanode.max.locked.memory configuration key to use the HDFS Lazy Persist feature. Its value limits the combined off-heap memory for blocks in RAM via caching and lazy persist writes.
+
+
+---
+
+* [HDFS-8332](https://issues.apache.org/jira/browse/HDFS-8332) | *Major* | **DFS client API calls should check filesystem closed**
+
+Users may need special attention for this change while upgrading to this version. Previously user could call some APIs(example: setReplication) wrongly even after closing the fs object. With this change DFS client will not allow any operations to call on closed fs objects. As calling fs operations on closed fs is not right thing to do, users need to correct the usage if any.
+
+
+---
+
+* [HADOOP-11698](https://issues.apache.org/jira/browse/HADOOP-11698) | *Major* | **Remove DistCpV1 and Logalyzer**
+
+Removed DistCpV1 and Logalyzer.
+
+
+---
+
+* [HADOOP-11772](https://issues.apache.org/jira/browse/HADOOP-11772) | *Major* | **RPC Invoker relies on static ClientCache which has synchronized(this) blocks**
+
+The Client#call() methods that are deprecated since 0.23 have been removed.
+
+
+---
+
+* [YARN-3684](https://issues.apache.org/jira/browse/YARN-3684) | *Major* | **Change ContainerExecutor's primary lifecycle methods to use a more extensible mechanism for passing information.**
+
+Modifying key methods in ContainerExecutor to use context objects instead of an argument list. This is more extensible and less brittle.
+
+
+---
+
+* [YARN-2336](https://issues.apache.org/jira/browse/YARN-2336) | *Major* | **Fair scheduler REST api returns a missing '[' bracket JSON for deep queue tree**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [HDFS-8135](https://issues.apache.org/jira/browse/HDFS-8135) | *Major* | **Remove the deprecated FSConstants class**
+
+The FSConstants class has been deprecated since 0.23 and it is removed in the release.
+
+
+---
+
+* [MAPREDUCE-6336](https://issues.apache.org/jira/browse/MAPREDUCE-6336) | *Major* | **Enable v2 FileOutputCommitter by default**
+
+mapreduce.fileoutputcommitter.algorithm.version now defaults to 2.
+
+In algorithm version 1:
+
+ 1. commitTask renames directory
+ $joboutput/\_temporary/$appAttemptID/\_temporary/$taskAttemptID/
+ to
+ $joboutput/\_temporary/$appAttemptID/$taskID/
+
+ 2. recoverTask renames
+ $joboutput/\_temporary/$appAttemptID/$taskID/
+ to
+ $joboutput/\_temporary/($appAttemptID + 1)/$taskID/
+
+ 3. commitJob merges every task output file in
+ $joboutput/\_temporary/$appAttemptID/$taskID/
+ to
+ $joboutput/, then it will delete $joboutput/\_temporary/
+ and write $joboutput/\_SUCCESS
+
+commitJob's run time, number of RPC, is O(n) in terms of output files, which is discussed in MAPREDUCE-4815, and can take minutes.
+
+Algorithm version 2 changes the behavior of commitTask, recoverTask, and commitJob.
+
+ 1. commitTask renames all files in
+ $joboutput/\_temporary/$appAttemptID/\_temporary/$taskAttemptID/
+ to $joboutput/
+
+ 2. recoverTask is a nop strictly speaking, but for
+ upgrade from version 1 to version 2 case, it checks if there
+ are any files in
+ $joboutput/\_temporary/($appAttemptID - 1)/$taskID/
+ and renames them to $joboutput/
+
+ 3. commitJob deletes $joboutput/\_temporary and writes
+ $joboutput/\_SUCCESS
+
+Algorithm 2 takes advantage of task parallelism and makes commitJob itself O(1). However, the window of vulnerability for having incomplete output in $jobOutput directory is much larger. Therefore, pipeline logic for consuming job outputs should be built on checking for existence of \_SUCCESS marker.
+
+
+---
+
+* [YARN-2355](https://issues.apache.org/jira/browse/YARN-2355) | *Major* | **MAX\_APP\_ATTEMPTS\_ENV may no longer be a useful env var for a container**
+
+Removed consumption of the MAX\_APP\_ATTEMPTS\_ENV environment variable
+
+
+---
+
+* [HDFS-5033](https://issues.apache.org/jira/browse/HDFS-5033) | *Minor* | **Bad error message for fs -put/copyFromLocal if user doesn't have permissions to read the source**
+
+"Permission denied" error message when unable to read local file for -put/copyFromLocal
+
+
+---
+
+* [HADOOP-9905](https://issues.apache.org/jira/browse/HADOOP-9905) | *Major* | **remove dependency of zookeeper for hadoop-client**
+
+Zookeeper jar removed from hadoop-client dependency tree.
+
+
+---
+
+* [HDFS-8486](https://issues.apache.org/jira/browse/HDFS-8486) | *Blocker* | **DN startup may cause severe data loss**
+
+<!-- markdown -->
+Public service notice:
+* Every restart of a 2.6.x or 2.7.0 DN incurs a risk of unwanted block deletion.
+* Apply this patch if you are running a pre-2.7.1 release.
+
+
+---
+
+* [HDFS-8270](https://issues.apache.org/jira/browse/HDFS-8270) | *Major* | **create() always retried with hardcoded timeout when file already exists with open lease**
+
+Proxy level retries will not be done on AlreadyBeingCreatedExeption for create() op.
+
+
+---
+
+* [YARN-41](https://issues.apache.org/jira/browse/YARN-41) | *Major* | **The RM should handle the graceful shutdown of the NM.**
+
+The behavior of shutdown a NM could be different (if NM work preserving is not enabled): NM will unregister to RM immediately rather than waiting for timeout to be LOST. A new status of NodeStatus - SHUTDOWN is involved which could affect UI, CLI and ClusterMetrics for node's status.
+
+
+---
+
+* [HDFS-8591](https://issues.apache.org/jira/browse/HDFS-8591) | *Minor* | **Remove support for deprecated configuration key dfs.namenode.decommission.nodes.per.interval**
+
+Related to the decommission enhancements in HDFS-7411, this change removes the deprecated configuration key "dfs.namenode.decommission.nodes.per.interval" which has been subsumed by the configuration key "dfs.namenode.decommission.blocks.per.interval".
+
+
+---
+
+* [HADOOP-7139](https://issues.apache.org/jira/browse/HADOOP-7139) | *Major* | **Allow appending to existing SequenceFiles**
+
+Existing sequence files can be appended.
+
+
+---
+
+* [HDFS-8582](https://issues.apache.org/jira/browse/HDFS-8582) | *Minor* | **Support getting a list of reconfigurable config properties and do not generate spurious reconfig warnings**
+
+Add a new option "properties" to the "dfsadmin -reconfig" command to get a list of reconfigurable properties.
+
+
+---
+
+* [HDFS-6564](https://issues.apache.org/jira/browse/HDFS-6564) | *Major* | **Use slf4j instead of common-logging in hdfs-client**
+
+Users may need special attention for this change while upgrading to this version. Previously hdfs client was using commons-logging as the logging framework. With this change it will use slf4j framework. For more details about slf4j, please see: http://www.slf4j.org/manual.html. Also, org.apache.hadoop.hdfs.protocol.CachePoolInfo#LOG public static member variable has been removed as it is not used anywhere. Users need to correct their code if any one has a reference to this variable. One can retrieve the named logger via the logging framework of their choice directly like, org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(org.apache.hadoop.hdfs.protocol.CachePoolInfo.class);
+
+
+---
+
+* [HDFS-6440](https://issues.apache.org/jira/browse/HDFS-6440) | *Major* | **Support more than 2 NameNodes**
+
+This feature adds support for running additional standby NameNodes, which provides additional fault-tolerance. It is designed for a total of 3-5 NameNodes.
+
+
+---
+
+* [YARN-3823](https://issues.apache.org/jira/browse/YARN-3823) | *Minor* | **Fix mismatch in default values for yarn.scheduler.maximum-allocation-vcores property**
+
+Default value for 'yarn.scheduler.maximum-allocation-vcores' changed from 32 to 4.
+
+
+---
+
+* [HADOOP-5732](https://issues.apache.org/jira/browse/HADOOP-5732) | *Minor* | **Add SFTP FileSystem**
+
+Added SFTP filesystem by using the JSch library.
+
+
+---
+
+* [YARN-3069](https://issues.apache.org/jira/browse/YARN-3069) | *Major* | **Document missing properties in yarn-default.xml**
+
+Documented missing properties and added the regression test to verify that there are no missing properties in yarn-default.xml.
+
+
+---
+
+* [MAPREDUCE-6427](https://issues.apache.org/jira/browse/MAPREDUCE-6427) | *Minor* | **Fix typo in JobHistoryEventHandler**
+
+There is a typo in the event string "WORKFLOW\_ID" (as "WORKLFOW\_ID"). The branch-2 change will publish both event strings for compatibility with consumers, but the misspelled metric will be removed in trunk.
+
+
+---
+
+* [HDFS-7582](https://issues.apache.org/jira/browse/HDFS-7582) | *Major* | **Enforce maximum number of ACL entries separately per access and default.**
+
+Limit on Maximum number of ACL entries(32) will be enforced separately on access and default ACLs. So in total, max. 64 ACL entries can be present in a ACL spec.
+
+
+---
+
+* [HADOOP-12269](https://issues.apache.org/jira/browse/HADOOP-12269) | *Major* | **Update aws-sdk dependency to 1.10.6; move to aws-sdk-s3**
+
+The Maven dependency on aws-sdk has been changed to aws-sdk-s3 and the version bumped. Applications depending on transitive dependencies pulled in by aws-sdk and not aws-sdk-s3 might not work.
+
+
+---
+
+* [HDFS-8818](https://issues.apache.org/jira/browse/HDFS-8818) | *Major* | **Allow Balancer to run faster**
+
+Add a new conf "dfs.balancer.max-size-to-move" so that Balancer.MAX\_SIZE\_TO\_MOVE becomes configurable.
+
+
+---
+
+* [HDFS-8895](https://issues.apache.org/jira/browse/HDFS-8895) | *Major* | **Remove deprecated BlockStorageLocation APIs**
+
+This removes the deprecated DistributedFileSystem#getFileBlockStorageLocations API used for getting VolumeIds of block replicas. Applications interested in the volume of a replica can instead consult BlockLocation#getStorageIds to obtain equivalent information.
+
+
+---
+
+* [HADOOP-12352](https://issues.apache.org/jira/browse/HADOOP-12352) | *Trivial* | **Delay in checkpointing Trash can leave trash for 2 intervals before deleting**
+
+Fixes an Trash related issue wherein a delay in the periodic checkpointing of one user's directory causes the subsequent user directory checkpoints to carry a newer timestamp, thereby delaying their eventual deletion.
+
+
+---
+
+* [HDFS-8900](https://issues.apache.org/jira/browse/HDFS-8900) | *Major* | **Compact XAttrs to optimize memory footprint.**
+
+The config key "dfs.namenode.fs-limits.max-xattr-size" can no longer be set to a value of 0 (previously used to indicate unlimited) or a value greater than 32KB. This is a constraint on xattr size similar to many local filesystems.
+
+
+---
+
+* [HDFS-8890](https://issues.apache.org/jira/browse/HDFS-8890) | *Major* | **Allow admin to specify which blockpools the balancer should run on**
+
+Adds a new blockpools flag to the balancer. This allows admins to specify which blockpools the balancer will run on.
+Usage:
+-blockpools \<comma-separated list of blockpool ids\>
+The balancer will only run on blockpools included in this list.
+
+
+---
+
+* [HDFS-8981](https://issues.apache.org/jira/browse/HDFS-8981) | *Minor* | **Adding revision to data node jmx getVersion() method**
+
+getSoftwareVersion method would replace original getVersion method, which returns the version string.
+
+The new getVersion method would return both version string and revision string.
+
+
+---
+
+* [YARN-4087](https://issues.apache.org/jira/browse/YARN-4087) | *Major* | **Followup fixes after YARN-2019 regarding RM behavior when state-store error occurs**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [HADOOP-12384](https://issues.apache.org/jira/browse/HADOOP-12384) | *Major* | **Add "-direct" flag option for fs copy so that user can choose not to create ".\_COPYING\_" file**
+
+An option '-d' added for all command-line copy commands to skip intermediate '.COPYING' file creation.
+
+
+---
+
+* [HDFS-8929](https://issues.apache.org/jira/browse/HDFS-8929) | *Major* | **Add a metric to expose the timestamp of the last journal**
+
+Exposed a metric 'LastJournalTimestamp' for JournalNode
+
+
+---
+
+* [HDFS-7116](https://issues.apache.org/jira/browse/HDFS-7116) | *Major* | **Add a command to get the balancer bandwidth**
+
+Exposed command "-getBalancerBandwidth" in dfsadmin to get the bandwidth of balancer.
+
+
+---
+
+* [HDFS-8829](https://issues.apache.org/jira/browse/HDFS-8829) | *Major* | **Make SO\_RCVBUF and SO\_SNDBUF size configurable for DataTransferProtocol sockets and allow configuring auto-tuning**
+
+HDFS-8829 introduces two new configuration settings: dfs.datanode.transfer.socket.send.buffer.size and dfs.datanode.transfer.socket.recv.buffer.size. These settings can be used to control the socket send buffer and receive buffer sizes respectively on the DataNode for client-DataNode and DataNode-DataNode connections. The default values of both settings are 128KB for backwards compatibility. For optimum performance it is recommended to set these values to zero to enable the OS networking stack to auto-tune buffer sizes.
+
+
+---
+
+* [YARN-313](https://issues.apache.org/jira/browse/YARN-313) | *Critical* | **Add Admin API for supporting node resource configuration in command line**
+
+After this patch, the feature to support NM resource dynamically configuration is completed, so that user can configure NM with new resource without bring NM down or decommissioned.
+Two CLIs are provided to support update resources on individual node or a batch of nodes:
+1. Update resource on single node: yarn rmadmin -updateNodeResource [NodeID] [MemSize] [vCores]
+2. Update resource on a batch of nodes: yarn rmadmin -refreshNodesResources, that reflect nodes' resource configuration defined in dynamic-resources.xml which is loaded by RM dynamically (like capacity-scheduler.xml or fair-scheduler.xml).
+The first version of configuration format is:
+\<configuration\>
+ \<property\>
+ \<name\>yarn.resource.dynamic.nodes\</name\>
+ \<value\>h1:1234\</value\>
+ \</property\>
+ \<property\>
+ \<name\>yarn.resource.dynamic.h1:1234.vcores\</name\>
+ \<value\>16\</value\>
+ \</property\>
+ \<property\>
+ \<name\>yarn.resource.dynamic.h1:1234.memory\</name\>
+ \<value\>1024\</value\>
+ \</property\>
+\</configuration\>
+
+
+---
+
+* [HADOOP-12416](https://issues.apache.org/jira/browse/HADOOP-12416) | *Major* | **Trash messages should be handled by Logger instead of being delivered on System.out**
+
+Now trash message is not printed to System.out. It is handled by Logger instead.
+
+
+---
+
+* [HADOOP-12360](https://issues.apache.org/jira/browse/HADOOP-12360) | *Minor* | **Create StatsD metrics2 sink**
+
+Added StatsD metrics2 sink
+
+
+---
+
+* [HDFS-9013](https://issues.apache.org/jira/browse/HDFS-9013) | *Major* | **Deprecate NameNodeMXBean#getNNStarted in branch2 and remove from trunk**
+
+NameNodeMXBean#getNNStarted() metric is deprecated in branch-2 and removed from trunk.
+
+
+---
+
+* [HADOOP-12437](https://issues.apache.org/jira/browse/HADOOP-12437) | *Major* | **Allow SecurityUtil to lookup alternate hostnames**
+
+HADOOP-12437 introduces two new configuration settings: hadoop.security.dns.interface and hadoop.security.dns.nameserver. These settings can be used to control how Hadoop service instances look up their own hostname and may be required in some multi-homed environments where hosts are configured with multiple hostnames in DNS or hosts files. They supersede the existing settings dfs.datanode.dns.interface and dfs.datanode.dns.nameserver.
+
+
+---
+
+* [HADOOP-12446](https://issues.apache.org/jira/browse/HADOOP-12446) | *Major* | **Undeprecate createNonRecursive()**
+
+FileSystem#createNonRecursive() is undeprecated.
+
+
+---
+
+* [HDFS-8696](https://issues.apache.org/jira/browse/HDFS-8696) | *Major* | **Make the lower and higher watermark in the DN Netty server configurable**
+
+Introduced two new configuration dfs.webhdfs.netty.low.watermark and dfs.webhdfs.netty.high.watermark to enable tuning the size of the buffers of the Netty server inside Datanodes.
+
+
+---
+
+* [HDFS-7285](https://issues.apache.org/jira/browse/HDFS-7285) | *Major* | **Erasure Coding Support inside HDFS**
+
+<!-- markdown -->
+HDFS now provides native support for erasure coding (EC) to store data more efficiently. Each individual directory can be configured with an EC policy with command `hdfs erasurecode -setPolicy`. When a file is created, it will inherit the EC policy from its nearest ancestor directory to determine how its blocks are stored. Compared to 3-way replication, the default EC policy saves 50% of storage space while also tolerating more storage failures.
+
+To support small files, the currently phase of HDFS-EC stores blocks in _striped_ layout, where a logical file block is divided into small units (64KB by default) and distributed to a set of DataNodes. This enables parallel I/O but also decreases data locality. Therefore, the cluster environment and I/O workloads should be considered before configuring EC policies.
+
+
+---
+
+* [HDFS-9085](https://issues.apache.org/jira/browse/HDFS-9085) | *Trivial* | **Show renewer information in DelegationTokenIdentifier#toString**
+
+The output of the "hdfs fetchdt --print" command now includes the token renewer appended to the end of the existing token information. This change may be incompatible with tools that parse the output of the command.
+
+
+---
+
+* [HADOOP-12493](https://issues.apache.org/jira/browse/HADOOP-12493) | *Major* | **bash unit tests are failing**
+
+In the extremely rare event that HADOOP\_USER\_IDENT and USER environment variables are not defined, we now fall back to use 'hadoop' as the identification string.
+
+
+---
+
+* [HADOOP-12495](https://issues.apache.org/jira/browse/HADOOP-12495) | *Major* | **Fix posix\_spawn error on OS X**
+
+When Hadoop JVMs create other processes on OS X, it will always use posix\_spawn.
+
+
+---
+
+* [HDFS-9070](https://issues.apache.org/jira/browse/HDFS-9070) | *Major* | **Allow fsck display pending replica location information for being-written blocks**
+
+The output of fsck command for being written hdfs files had been changed. When using fsck against being written hdfs files with {{-openforwrite}} and {{-files -blocks -locations}}, the fsck output will include the being written block for replication files or being written block group for erasure code files.
+
+
+---
+
+* [HDFS-9278](https://issues.apache.org/jira/browse/HDFS-9278) | *Trivial* | **Fix preferredBlockSize typo in OIV XML output**
+
+The preferred block size XML element has been corrected from "\\<perferredBlockSize\>" to "\\<preferredBlockSize\>".
+
+
+---
+
+* [HDFS-9184](https://issues.apache.org/jira/browse/HDFS-9184) | *Major* | **Logging HDFS operation's caller context into audit logs**
+
+The feature needs to enabled by setting "hadoop.caller.context.enabled" to true. When the feature is used, additional fields are written into namenode audit log records.
+
+
+---
+
+* [HDFS-9259](https://issues.apache.org/jira/browse/HDFS-9259) | *Major* | **Make SO\_SNDBUF size configurable at DFSClient side for hdfs write scenario**
+
+Introduces a new configuration setting dfs.client.socket.send.buffer.size to control the socket send buffer size for writes. Setting it to zero enables TCP auto-tuning on systems that support it.
+
+
+---
+
+* [HDFS-9311](https://issues.apache.org/jira/browse/HDFS-9311) | *Major* | **Support optional offload of NameNode HA service health checks to a separate RPC server.**
+
+There is now support for offloading HA health check RPC activity to a separate RPC server endpoint running within the NameNode process. This may improve reliability of HA health checks and prevent spurious failovers in highly overloaded conditions. For more details, please refer to the hdfs-default.xml documentation for properties dfs.namenode.lifeline.rpc-address, dfs.namenode.lifeline.rpc-bind-host and dfs.namenode.lifeline.handler.count.
+
+
+---
+
+* [HDFS-6200](https://issues.apache.org/jira/browse/HDFS-6200) | *Major* | **Create a separate jar for hdfs-client**
+
+Projects that access HDFS can depend on the hadoop-hdfs-client module instead of the hadoop-hdfs module to avoid pulling in unnecessary dependency.
+
+
+---
+
+* [HADOOP-10787](https://issues.apache.org/jira/browse/HADOOP-10787) | *Blocker* | **Rename/remove non-HADOOP\_\*, etc from the shell scripts**
+
+<!-- markdown -->
+The following shell environment variables have been deprecated:
+
+| Old | New |
+|:---- |:---- |
+| DEFAULT\_LIBEXEC\_DIR | HADOOP\_DEFAULT\_LIBEXEC\_DIR |
+| SLAVE\_NAMES | HADOOP\_SLAVE\_NAMES |
+| TOOL\_PATH | HADOOP\_TOOLS\_PATH |
+
+In addition:
+
+* DEFAULT\_LIBEXEC\_DIR will NOT be automatically transitioned to HADOOP\_DEFAULT\_LIBEXEC\_DIR and will require changes to any scripts setting that value. A warning will be printed to the screen if DEFAULT\_LIBEXEC\_DIR has been configured.
+* HADOOP\_TOOLS\_PATH is now properly handled as a multi-valued, Java classpath-style variable. Prior, multiple values assigned to TOOL\_PATH would not work a predictable manner.
+
+
+---
+
+* [HDFS-9057](https://issues.apache.org/jira/browse/HDFS-9057) | *Major* | **allow/disallow snapshots via webhdfs**
+
+Snapshots can be allowed/disallowed on a directory via WebHdfs from users with superuser privilege.
+
+
+---
+
+* [MAPREDUCE-5485](https://issues.apache.org/jira/browse/MAPREDUCE-5485) | *Critical* | **Allow repeating job commit by extending OutputCommitter API**
+
+Previously, the MR job will get failed if AM get restarted for some reason (like node failure, etc.) during its doing commit job no matter if AM attempts reach to the maximum attempts.
+In this improvement, we add a new API isCommitJobRepeatable() to OutputCommitter interface which to indicate if job's committer can do commitJob again if previous commit work is interrupted by NM/AM failures, etc. The instance of OutputCommitter, which support repeatable job commit (like FileOutputCommitter in algorithm 2), can allow AM to continue the commitJob() after AM restart as a new attempt.
+
+
+---
+
+* [HADOOP-12294](https://issues.apache.org/jira/browse/HADOOP-12294) | *Major* | **Throw an Exception when fs.permissions.umask-mode is misconfigured**
+
+The support of the deprecated dfs.umask key is removed in Hadoop 3.0.
+
+
+---
+
+* [HADOOP-10465](https://issues.apache.org/jira/browse/HADOOP-10465) | *Minor* | **Fix use of generics within SortedMapWritable**
+
+SortedMapWritable has changed to SortedMapWritable\<K extends WritableComparable\<? super K\>\>. That way user can declare the class by such as SortedMapWritable\<Text\>.
+
+
+---
+
+* [HADOOP-12313](https://issues.apache.org/jira/browse/HADOOP-12313) | *Critical* | **NPE in JvmPauseMonitor when calling stop() before start()**
+
+Allow stop() before start() completed in JvmPauseMonitor
+
+
+---
+
+* [HDFS-5165](https://issues.apache.org/jira/browse/HDFS-5165) | *Minor* | **Remove the TotalFiles metrics**
+
+Now TotalFiles metric is removed from FSNameSystem. Use FilesTotal instead.
+
+
+---
+
+* [HDFS-8335](https://issues.apache.org/jira/browse/HDFS-8335) | *Major* | **FSNamesystem should construct FSPermissionChecker only if permission is enabled**
+
+Only check permissions when permissions enabled in FSDirStatAndListingOp.getFileInfo() and getListingInt()
+
+
+---
+
+* [HDFS-8831](https://issues.apache.org/jira/browse/HDFS-8831) | *Major* | **Trash Support for deletion in HDFS encryption zone**
+
+Add Trash support for deleting files within encryption zones. Deleted files will remain encrypted and they will be moved to a \u201c.Trash\u201d subdirectory under the root of the encryption zone, prefixed by $USER/current. Checkpoint and expunge continue to work like the existing Trash.
+
+
+---
+
+* [HDFS-9214](https://issues.apache.org/jira/browse/HDFS-9214) | *Major* | **Support reconfiguring dfs.datanode.balance.max.concurrent.moves without DN restart**
+
+Steps to reconfigure:
+1. change value of the parameter in corresponding xml configuration file
+2. to reconfigure, run
+ hdfs dfsadmin -reconfig datanode \<dn\_addr\>:\<ipc\_port\> start
+3. repeat step 2 until all DNs are reconfigured
+4. to check status of the most recent reconfigure operation, run
+ hdfs dfsadmin -reconfig datanode \<dn\_addr\>:\<ipc\_port\> status
+5. to query a list reconfigurable properties on DN, run
+ hdfs dfsadmin -reconfig datanode \<dn\_addr\>:\<ipc\_port\> properties
+
+
+---
+
+* [YARN-3623](https://issues.apache.org/jira/browse/YARN-3623) | *Major* | **We should have a config to indicate the Timeline Service version**
+
+Add a new configuration "yarn.timeline-service.version" to indicate what is the current version of the running timeline service. For example, if "yarn.timeline-service.version" is 1.5, and "yarn.timeline-service.enabled" is true, it means the cluster will and should bring up the timeline service v.1.5. On the client side, if the client uses the same version of timeline service, it should succeed. If the client chooses to use a smaller version in spite of this, then depending on how robust the compatibility story is between versions, the results may vary.
+
+
+---
+
+* [YARN-4207](https://issues.apache.org/jira/browse/YARN-4207) | *Major* | **Add a non-judgemental YARN app completion status**
+
+Adds the ENDED attribute to o.a.h.yarn.api.records.FinalApplicationStatus
+
+
+---
+
+* [HADOOP-12657](https://issues.apache.org/jira/browse/HADOOP-12657) | *Minor* | **Add a option to skip newline on empty files with getMerge -nl**
+
+Added -skip-empty-file option to hadoop fs -getmerge command. With the option, delimiter (LF) is not printed for empty files even if -nl option is used.
+
+
+---
+
+* [HDFS-9047](https://issues.apache.org/jira/browse/HDFS-9047) | *Major* | **Retire libwebhdfs**
+
+libwebhdfs has been retired in 2.8.0 due to the lack of maintenance.
+
+
+---
+
+* [HADOOP-11262](https://issues.apache.org/jira/browse/HADOOP-11262) | *Major* | **Enable YARN to use S3A**
+
+S3A has been made accessible through the FileContext API.
+
+
+---
+
+* [HADOOP-12635](https://issues.apache.org/jira/browse/HADOOP-12635) | *Major* | **Adding Append API support for WASB**
+
+The Azure Blob Storage file system (WASB) now includes optional support for use of the append API by a single writer on a path. Please note that the implementation differs from the semantics of HDFS append. HDFS append internally guarantees that only a single writer may append to a path at a given time. WASB does not enforce this guarantee internally. Instead, the application must enforce access by a single writer, such as by running single-threaded or relying on some external locking mechanism to coordinate concurrent processes. Refer to the Azure Blob Storage documentation page for more details on enabling append in configuration.
+
+
+---
+
+* [HADOOP-12651](https://issues.apache.org/jira/browse/HADOOP-12651) | *Major* | **Replace dev-support with wrappers to Yetus**
+
+<!-- markdown -->
+
+* Major portions of dev-support have been replaced with wrappers to Apache Yetus:
+ * releasedocmaker.py is now dev-support/bin/releasedocmaker
+ * shelldocs.py is now dev-support/bin/shelldocs
+ * smart-apply-patch.sh is now dev-support/bin/smart-apply-patch
+ * test-patch.sh is now dev-support/bin/test-patch
+* See the dev-support/README.md file for more details on how to control the wrappers to various degrees.
+
+
+---
+
+* [HDFS-9525](https://issues.apache.org/jira/browse/HDFS-9525) | *Blocker* | **hadoop utilities need to support provided delegation tokens**
+
+If hadoop.token.files property is defined and configured to one or more comma-delimited delegation token files, Hadoop will use those token files to connect to the services as named in the token.
+
+
+---
+
+* [HDFS-9503](https://issues.apache.org/jira/browse/HDFS-9503) | *Major* | **Replace -namenode option with -fs for NNThroughputBenchmark**
+
+The patch replaces -namenode option with -fs for specifying the remote name node against which the benchmark is running. Before this patch, if '-namenode' was not given, the benchmark would run in standalone mode, ignoring the 'fs.defaultFS' in config file even if it's remote. With this patch, the benchmark, as other tools, will rely on the 'fs.defaultFS' config, which is overridable by -fs command option, to run standalone mode or remote mode.
+
+
+---
+
+* [HADOOP-12426](https://issues.apache.org/jira/browse/HADOOP-12426) | *Minor* | **Add Entry point for Kerberos health check**
+
+Hadoop now includes a shell command named KDiag that helps with diagnosis of Kerberos misconfiguration problems. Please refer to the Secure Mode documentation for full details on usage of the command.
+
+
+---
+
+* [HADOOP-12805](https://issues.apache.org/jira/browse/HADOOP-12805) | *Major* | **Annotate CanUnbuffer with @InterfaceAudience.Public**
+
+Made CanBuffer interface public for use in client applications.
+
+
+---
+
+* [HADOOP-12548](https://issues.apache.org/jira/browse/HADOOP-12548) | *Major* | **Read s3a creds from a Credential Provider**
+
+The S3A Hadoop-compatible file system now support reading its S3 credentials from the Hadoop Credential Provider API in addition to XML configuration files.
+
+
+---
+
+* [HDFS-9711](https://issues.apache.org/jira/browse/HDFS-9711) | *Major* | **Integrate CSRF prevention filter in WebHDFS.**
+
+WebHDFS now supports options to enforce cross-site request forgery (CSRF) prevention for HTTP requests to both the NameNode and the DataNode. Please refer to the updated WebHDFS documentation for a description of this feature and further details on how to configure it.
+
+
+---
+
+* [HADOOP-12794](https://issues.apache.org/jira/browse/HADOOP-12794) | *Major* | **Support additional compression levels for GzipCodec**
+
+Added New compression levels for GzipCodec that can be set in zlib.compress.level
+
+
+---
+
+* [MAPREDUCE-6613](https://issues.apache.org/jira/browse/MAPREDUCE-6613) | *Minor* | **Change mapreduce.jobhistory.jhist.format default from json to binary**
+
+Default of 'mapreduce.jobhistory.jhist.format' property changed from 'json' to 'binary'. Creates smaller, binary Avro .jhist files for faster JHS performance.
+
+
+---
+
+* [HDFS-9425](https://issues.apache.org/jira/browse/HDFS-9425) | *Major* | **Expose number of blocks per volume as a metric**
+
+Number of blocks per volume is made available as a metric.
+
+
+---
+
+* [HADOOP-12668](https://issues.apache.org/jira/browse/HADOOP-12668) | *Critical* | **Support excluding weak Ciphers in HttpServer2 through ssl-server.conf**
+
+The Code Changes include following:
+- Modified DFSUtil.java in Apache HDFS project for supplying new parameter ssl.server.exclude.cipher.list
+- Modified HttpServer2.java in Apache Hadoop-common project to work with new parameter and exclude ciphers using jetty setExcludeCihers method.
+- Modfied associated test classes to owrk with existing code and also cover the newfunctionality in junit
+
+
+---
+
+* [HADOOP-12555](https://issues.apache.org/jira/browse/HADOOP-12555) | *Minor* | **WASB to read credentials from a credential provider**
+
+The hadoop-azure file system now supports configuration of the Azure Storage account credentials using the standard Hadoop Credential Provider API. For details, please refer to the documentation on hadoop-azure and the Credential Provider API.
+
+
+---
+
+* [HDFS-9395](https://issues.apache.org/jira/browse/HDFS-9395) | *Major* | **Make HDFS audit logging consistant**
+
+<!-- markdown -->
+
+Audit logs will now only be generated in the following two cases:
+* When an operation results in an `AccessControlException`
+* When an operation is successful
+
+Notably, this means audit log events will not be generated for exceptions besides AccessControlException.
+
+
+---
+
+* [MAPREDUCE-6622](https://issues.apache.org/jira/browse/MAPREDUCE-6622) | *Critical* | **Add capability to set JHS job cache to a task-based limit**
+
+Two recommendations for the mapreduce.jobhistory.loadedtasks.cache.size property:
+1) For every 100k of cache size, set the heap size of the Job History Server to 1.2GB. For example, mapreduce.jobhistory.loadedtasks.cache.size=500000, heap size=6GB.
+2) Make sure that the cache size is larger than the number of tasks required for the largest job run on the cluster. It might be a good idea to set the value slightly higher (say, 20%) in order to allow for job size growth.
+
+
+---
+
+* [HADOOP-12552](https://issues.apache.org/jira/browse/HADOOP-12552) | *Minor* | **Fix undeclared/unused dependency to httpclient**
+
+Dependency on commons-httpclient::commons-httpclient was removed from hadoop-common. Downstream projects using commons-httpclient transitively provided by hadoop-common need to add explicit dependency to their pom. Since commons-httpclient is EOL, it is recommended to migrate to org.apache.httpcomponents:httpclient which is the successor.
+
+
+---
+
+* [HADOOP-12850](https://issues.apache.org/jira/browse/HADOOP-12850) | *Major* | **pull shell code out of hadoop-dist**
+
+This change contains the content of HADOOP-10115 which is an incompatible change.
+
+
+---
+
+* [HDFS-8791](https://issues.apache.org/jira/browse/HDFS-8791) | *Blocker* | **block ID-based DN storage layout can be very slow for datanode on ext4**
+
+HDFS-8791 introduces a new datanode layout format. This layout is identical to the previous block id based layout except it has a smaller 32x32 sub-directory structure in each data storage. On startup, the datanode will automatically upgrade it's storages to this new layout. Currently, datanode layout changes support rolling upgrades, on the other hand downgrading is not supported between datanode layout changes and a rollback would be required.
+
+
+---
+
+* [HDFS-9887](https://issues.apache.org/jira/browse/HDFS-9887) | *Major* | **WebHdfs socket timeouts should be configurable**
+
+Added new configuration options: dfs.webhdfs.socket.connect-timeout and dfs.webhdfs.socket.read-timeout both defaulting to 60s.
+
+
+---
+
+* [HADOOP-11792](https://issues.apache.org/jira/browse/HADOOP-11792) | *Major* | **Remove all of the CHANGES.txt files**
+
+With the introduction of the markdown-formatted and automatically built changes file, the CHANGES.txt files have been eliminated.
+
+
+---
+
+* [HDFS-9239](https://issues.apache.org/jira/browse/HDFS-9239) | *Major* | **DataNode Lifeline Protocol: an alternative protocol for reporting DataNode liveness**
+
+This release adds a new feature called the DataNode Lifeline Protocol. If configured, then DataNodes can report that they are still alive to the NameNode via a fallback protocol, separate from the existing heartbeat messages. This can prevent the NameNode from incorrectly marking DataNodes as stale or dead in highly overloaded clusters where heartbeat processing is suffering delays. For more information, please refer to the hdfs-default.xml documentation for several new configuration properties: dfs.namenode.lifeline.rpc-address, dfs.namenode.lifeline.rpc-bind-host, dfs.datanode.lifeline.interval.seconds, dfs.namenode.lifeline.handler.ratio and dfs.namenode.lifeline.handler.count.
+
+
+---
+
+* [YARN-4762](https://issues.apache.org/jira/browse/YARN-4762) | *Blocker* | **NMs failing on DelegatingLinuxContainerRuntime init with LCE on**
+
+Fixed CgroupHandler's creation and usage to avoid NodeManagers crashing when LinuxContainerExecutor is enabled.
+
+
+---
+
+* [HDFS-1477](https://issues.apache.org/jira/browse/HDFS-1477) | *Major* | **Support reconfiguring dfs.heartbeat.interval and dfs.namenode.heartbeat.recheck-interval without NN restart**
+
+Steps to reconfigure:
+1. change value of the parameter in corresponding xml configuration file
+2. to reconfigure, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> start
+3. to check status of the most recent reconfigure operation, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> status
+4. to query a list reconfigurable properties on NN, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> properties
+
+
+---
+
+* [YARN-4785](https://issues.apache.org/jira/browse/YARN-4785) | *Major* | **inconsistent value type of the "type" field for LeafQueueInfo in response of RM REST API - cluster/scheduler**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [YARN-4732](https://issues.apache.org/jira/browse/YARN-4732) | *Trivial* | **\*ProcessTree classes have too many whitespace issues**
+
+
+
+
+---
+
+* [HADOOP-12857](https://issues.apache.org/jira/browse/HADOOP-12857) | *Major* | **Rework hadoop-tools**
+
+<!-- markdown -->
+* Turning on optional things from the tools directory such as S3 support can now be done in hadoop-env.sh with the HADOOP\_OPTIONAL\_TOOLS environment variable without impacting the various user-facing CLASSPATH variables.
+* The tools directory is no longer pulled in blindly for any utilities that pull it in.
+* TOOL\_PATH / HADOOP\_TOOLS\_PATH has been broken apart and replaced with HADOOP\_TOOLS\_HOME, HADOOP\_TOOLS\_DIR and HADOOP\_TOOLS\_LIB\_JARS\_DIR to be consistent with the rest of Hadoop.
+
+
+---
+
+* [HDFS-9694](https://issues.apache.org/jira/browse/HDFS-9694) | *Major* | **Make existing DFSClient#getFileChecksum() work for striped blocks**
+
+Makes the getFileChecksum API works with striped layout EC files. Checksum computation done by block level in the distributed fashion. The current API does not support to compare the checksum generated with normal file and the checksum generated for the same file but in striped layout.
+
+
+---
+
+* [HDFS-9640](https://issues.apache.org/jira/browse/HDFS-9640) | *Major* | **Remove hsftp from DistCp in trunk**
+
+DistCp in Hadoop 3.0 no longer supports -mapredSSLConf option. Use global ssl-client.xml configuration file for swebhdfs file systems instead.
+
+
+---
+
+* [HDFS-9349](https://issues.apache.org/jira/browse/HDFS-9349) | *Major* | **Support reconfiguring fs.protected.directories without NN restart**
+
+Steps to reconfigure:
+1. change value of the parameter in corresponding xml configuration file
+2. to reconfigure, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> start
+3. to check status of the most recent reconfigure operation, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> status
+4. to query a list reconfigurable properties on NN, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> properties
+
+
+---
+
+* [HADOOP-11393](https://issues.apache.org/jira/browse/HADOOP-11393) | *Major* | **Revert HADOOP\_PREFIX, go back to HADOOP\_HOME**
+
+On Unix platforms, HADOOP\_PREFIX has been deprecated in favor of returning to HADOOP\_HOME as in prior Apache Hadoop releases.
+
+
+---
+
+* [HADOOP-12967](https://issues.apache.org/jira/browse/HADOOP-12967) | *Major* | **Remove FileUtil#copyMerge**
+
+Removed FileUtil.copyMerge.
+
+
+---
+
+* [MAPREDUCE-6670](https://issues.apache.org/jira/browse/MAPREDUCE-6670) | *Minor* | **TestJobListCache#testEviction sometimes fails on Windows with timeout**
+
+Backport the fix to 2.7 and 2.8
+
+
+---
+
+* [HDFS-9945](https://issues.apache.org/jira/browse/HDFS-9945) | *Major* | **Datanode command for evicting writers**
+
+This new dfsadmin command, evictWriters, stops active block writing activities on a data node. The affected writes will continue without the node after a write pipeline recovery. This is useful when data node decommissioning is blocked by slow writers. If issued against a non-decommissioing data node, all current writers will be stopped, but new write requests will continue to be served.
+
+
+---
+
+* [HADOOP-12963](https://issues.apache.org/jira/browse/HADOOP-12963) | *Minor* | **Allow using path style addressing for accessing the s3 endpoint**
+
+Add new flag to allow supporting path style addressing for s3a
+
+
+---
+
+* [HADOOP-12811](https://issues.apache.org/jira/browse/HADOOP-12811) | *Critical* | **Change kms server port number which conflicts with HMaster port number**
+
+The default port for KMS service is now 9600. This is to avoid conflicts on the previous port 16000, which is also used by HMaster as the default port.
+
+
+---
+
+* [YARN-4784](https://issues.apache.org/jira/browse/YARN-4784) | *Major* | **Fairscheduler: defaultQueueSchedulingPolicy should not accept FIFO**
+
+Clusters cannot use FIFO policy as the defaultQueueSchedulingPolicy. Clusters with a single level of queues will have to explicitly set the policy to FIFO if that is desired.
+
+
+---
+
+* [HDFS-9427](https://issues.apache.org/jira/browse/HDFS-9427) | *Critical* | **HDFS should not default to ephemeral ports**
+
+The patch updates the HDFS default HTTP/RPC ports to non-ephemeral ports. The changes are listed below:
+Namenode ports: 50470 --\> 9871, 50070 --\> 9870, 8020 --\> 9820
+Secondary NN ports: 50091 --\> 9869, 50090 --\> 9868
+Datanode ports: 50020 --\> 9867, 50010 --\> 9866, 50475 --\> 9865, 50075 --\> 9864
+
+
+---
+
+* [HDFS-3702](https://issues.apache.org/jira/browse/HDFS-3702) | *Minor* | **Add an option for NOT writing the blocks locally if there is a datanode on the same box as the client**
+
+This patch will attempt to allocate all replicas to remote DataNodes, by adding local DataNode to the excluded DataNodes. If no sufficient replicas can be obtained, it will fall back to default block placement policy, which writes one replica to local DataNode.
+
+
+---
+
+* [HADOOP-12563](https://issues.apache.org/jira/browse/HADOOP-12563) | *Major* | **Updated utility to create/modify token files**
+
+This feature introduces a new command called "hadoop dtutil" which lets users request and download delegation tokens with certain attributes.
+
+
+---
+
+* [HADOOP-13045](https://issues.apache.org/jira/browse/HADOOP-13045) | *Major* | **hadoop\_add\_classpath is not working in .hadooprc**
+
+<!-- markdown -->
+With this change, the `.hadooprc` file is now processed after Apache Hadoop has been fully bootstrapped. This allows for usage of the Apache Hadoop Shell API. A new file, `.hadoop-env`, now provides the ability for end users to override `hadoop-env.sh`.
+
+
+---
+
+* [MAPREDUCE-6526](https://issues.apache.org/jira/browse/MAPREDUCE-6526) | *Blocker* | **Remove usage of metrics v1 from hadoop-mapreduce**
+
+LocalJobRunnerMetrics and ShuffleClientMetrics were updated to use Hadoop Metrics V2 framework.
+
+
+---
+
+* [HDFS-9902](https://issues.apache.org/jira/browse/HDFS-9902) | *Major* | **Support different values of dfs.datanode.du.reserved per storage type**
+
+Reserved space can be configured independently for different storage types for clusters with heterogeneous storage. The 'dfs.datanode.du.reserved' property name can be suffixed with a storage types (i.e. one of ssd, disk, archival or ram\_disk). e.g. reserved space for RAM\_DISK storage can be configured using the property 'dfs.datanode.du.reserved.ram\_disk'. If specific storage type reservation is not configured then the value specified by 'dfs.datanode.du.reserved' will be used for all volumes.
+
+
+---
+
+* [HADOOP-12504](https://issues.apache.org/jira/browse/HADOOP-12504) | *Blocker* | **Remove metrics v1**
+
+<!-- markdown -->
+* org.apache.hadoop.metrics package was removed. Use org.apache.hadoop.metrics2 package instead.
+* "/metrics" endpoint was removed. Use "/jmx" instead to see the metrics.
+
+
+---
+
+* [HDFS-10324](https://issues.apache.org/jira/browse/HDFS-10324) | *Major* | **Trash directory in an encryption zone should be pre-created with correct permissions**
+
+HDFS will create a ".Trash" subdirectory when creating a new encryption zone to support soft delete for files deleted within the encryption zone. A new "crypto -provisionTrash" command has been introduced to provision trash directories for encryption zones created with Apache Hadoop minor releases prior to 2.8.0.
+
+
+---
+
+* [HDFS-10337](https://issues.apache.org/jira/browse/HDFS-10337) | *Minor* | **OfflineEditsViewer stats option should print 0 instead of null for the count of operations**
+
+The output of "hdfs oev -p stats" has changed. The option prints 0 instead of null for the count of the operations that have never been executed.
+
+
+---
+
+* [HADOOP-10694](https://issues.apache.org/jira/browse/HADOOP-10694) | *Major* | **Remove synchronized input streams from Writable deserialization**
+
+Remove invisible synchronization primitives from DataInputBuffer
+
+
+---
+
+* [HADOOP-13122](https://issues.apache.org/jira/browse/HADOOP-13122) | *Minor* | **Customize User-Agent header sent in HTTP requests by S3A.**
+
+S3A now includes the current Hadoop version in the User-Agent string passed through the AWS SDK to the S3 service. Users also may include optional additional information to identify their application. See the documentation of configuration property fs.s3a.user.agent.prefix for further details.
+
+
+---
+
+* [HADOOP-11858](https://issues.apache.org/jira/browse/HADOOP-11858) | *Blocker* | **[JDK8] Set minimum version of Hadoop 3 to JDK 8**
+
+The minimum required JDK version for Hadoop has been increased from JDK7 to JDK8.
+
+
+---
+
+* [HADOOP-12930](https://issues.apache.org/jira/browse/HADOOP-12930) | *Critical* | **[Umbrella] Dynamic subcommands for hadoop shell scripts**
+
+It is now possible to add or modify the behavior of existing subcommands in the hadoop, hdfs, mapred, and yarn scripts. See the Unix Shell Guide for more information.
+
+
+---
+
+* [HADOOP-12782](https://issues.apache.org/jira/browse/HADOOP-12782) | *Major* | **Faster LDAP group name resolution with ActiveDirectory**
+
+If the user object returned by LDAP server has the user's group object DN (supported by Active Directory), Hadoop can reduce LDAP group mapping latency by setting hadoop.security.group.mapping.ldap.search.attr.memberof to memberOf.
+
+
+---
+
+* [HADOOP-12723](https://issues.apache.org/jira/browse/HADOOP-12723) | *Major* | **S3A: Add ability to plug in any AWSCredentialsProvider**
+
+Users can integrate a custom credential provider with S3A. See documentation of configuration property fs.s3a.aws.credentials.provider for further details.
+
+
+---
+
+* [MAPREDUCE-6607](https://issues.apache.org/jira/browse/MAPREDUCE-6607) | *Minor* | **Enable regex pattern matching when mapreduce.task.files.preserve.filepattern is set**
+
+Before this fix, the files in .staging directory are always preserved when mapreduce.task.files.preserve.filepattern is set. After this fix, the files in .staging directory are preserved if the name of the directory matches the regex pattern specified by mapreduce.task.files.preserve.filepattern.
+
+
+---
+
+* [YARN-5035](https://issues.apache.org/jira/browse/YARN-5035) | *Major* | **FairScheduler: Adjust maxAssign dynamically when assignMultiple is turned on**
+
+Introducing a new configuration "yarn.scheduler.fair.dynamic.max.assign" to dynamically determine the resources to assign per heartbeat when assignmultiple is turned on. When turned on, the scheduler allocates roughly half of the remaining resources overriding any max.assign settings configured. This is turned ON by default.
+
+
+---
+
+* [YARN-5132](https://issues.apache.org/jira/browse/YARN-5132) | *Critical* | **Exclude generated protobuf sources from YARN Javadoc build**
+
+Exclude javadocs for proto-generated java classes.
+
+
+---
+
+* [HADOOP-13105](https://issues.apache.org/jira/browse/HADOOP-13105) | *Major* | **Support timeouts in LDAP queries in LdapGroupsMapping.**
+
+This patch adds two new config keys for supporting timeouts in LDAP query operations. The property "hadoop.security.group.mapping.ldap.connection.timeout.ms" is the connection timeout (in milliseconds), within which period if the LDAP provider doesn't establish a connection, it will abort the connect attempt. The property "hadoop.security.group.mapping.ldap.read.timeout.ms" is the read timeout (in milliseconds), within which period if the LDAP provider doesn't get a LDAP response, it will abort the read attempt.
+
+
+---
+
+* [HADOOP-13155](https://issues.apache.org/jira/browse/HADOOP-13155) | *Major* | **Implement TokenRenewer to renew and cancel delegation tokens in KMS**
+
+Enables renewal and cancellation of KMS delegation tokens. hadoop.security.key.provider.path needs to be configured to reach the key provider.
+
+
+---
+
+* [HADOOP-12807](https://issues.apache.org/jira/browse/HADOOP-12807) | *Minor* | **S3AFileSystem should read AWS credentials from environment variables**
+
+Adds support to S3AFileSystem for reading AWS credentials from environment variables.
+
+
+---
+
+* [HDFS-10375](https://issues.apache.org/jira/browse/HDFS-10375) | *Trivial* | **Remove redundant TestMiniDFSCluster.testDualClusters**
+
+Remove redundent TestMiniDFSCluster.testDualClusters to save time.
+
+
+---
+
+* [HDFS-10220](https://issues.apache.org/jira/browse/HDFS-10220) | *Major* | **A large number of expired leases can make namenode unresponsive and cause failover**
+
+Two new configuration have been added "dfs.namenode.lease-recheck-interval-ms" and "dfs.namenode.max-lock-hold-to-release-lease-ms" to fine tune the duty cycle with which the Namenode recovers old leases.
+
+
+---
+
+* [HADOOP-13237](https://issues.apache.org/jira/browse/HADOOP-13237) | *Minor* | **s3a initialization against public bucket fails if caller lacks any credentials**
+
+S3A now supports read access to a public S3 bucket even if the client does not configure any AWS credentials. See the documentation of configuration property fs.s3a.aws.credentials.provider for further details.
+
+
+---
+
+* [HADOOP-12537](https://issues.apache.org/jira/browse/HADOOP-12537) | *Minor* | **S3A to support Amazon STS temporary credentials**
+
+S3A now supports use of AWS Security Token Service temporary credentials for authentication to S3. Refer to the documentation of configuration property fs.s3a.session.token for further details.
+
+
+---
+
+* [HADOOP-13175](https://issues.apache.org/jira/browse/HADOOP-13175) | *Major* | **Remove hadoop-ant from hadoop-tools**
+
+The hadoop-ant module in hadoop-tools has been removed.
+
+
+---
+
+* [HADOOP-12666](https://issues.apache.org/jira/browse/HADOOP-12666) | *Major* | **Support Microsoft Azure Data Lake - as a file system in Hadoop**
+
+Hadoop now supports integration with Azure Data Lake as an alternative Hadoop-compatible file system. Please refer to the Hadoop site documentation of Azure Data Lake for details on usage and configuration.
+
+
+---
+
+* [HDFS-9016](https://issues.apache.org/jira/browse/HDFS-9016) | *Major* | **Display upgrade domain information in fsck**
+
+New fsck option "-upgradedomains" has been added to display upgrade domains of any block.
+
+
+---
+
+* [HADOOP-12892](https://issues.apache.org/jira/browse/HADOOP-12892) | *Blocker* | **fix/rewrite create-release**
+
+This rewrites the release process with a new dev-support/bin/create-release script. See http://wiki.apache.org/hadoop/HowToRelease for updated instructions on how to use it.
+
+
+---
+
+* [HADOOP-3733](https://issues.apache.org/jira/browse/HADOOP-3733) | *Minor* | **"s3:" URLs break when Secret Key contains a slash, even if encoded**
+
+Allows userinfo component of URI authority to contain a slash (escaped as %2F). Especially useful for accessing AWS S3 with distcp or hadoop fs.
+
+
+---
+
+* [HADOOP-13242](https://issues.apache.org/jira/browse/HADOOP-13242) | *Major* | **Authenticate to Azure Data Lake using client ID and keys**
+
+Adds support for Azure ActiveDirectory tokens using client ID and keys
+
+
+---
+
+* [HADOOP-9613](https://issues.apache.org/jira/browse/HADOOP-9613) | *Major* | **[JDK8] Update jersey version to latest 1.x release**
+
+Upgrading Jersey and its related libraries:
+
+1. Upgrading jersey from 1.9 to 1.19
+2. Adding jersey-servlet 1.19
+3. Upgrading grizzly-http-servlet from 2.1.2 to 2.2.21
+4. Adding grizzly-http 2.2.21
+5. Adding grizzly-http-server 2.2.21
+
+After upgrading Jersey from 1.12 to 1.13, the root element whose content is empty collection is changed from null to empty object({}).
+
+
+---
+
+* [HDFS-10328](https://issues.apache.org/jira/browse/HDFS-10328) | *Minor* | **Add per-cache-pool default replication num configuration**
+
+Add per-cache-pool default replication num configuration
+
+
+---
+
+* [HADOOP-13203](https://issues.apache.org/jira/browse/HADOOP-13203) | *Major* | **S3A: Support fadvise "random" mode for high performance readPositioned() reads**
+
+S3A has added support for configurable input policies. Similar to fadvise, this configuration provides applications with a way to specify their expected access pattern (sequential or random) while reading a file. S3A then performs optimizations tailored to that access pattern. See site documentation of the fs.s3a.experimental.input.fadvise configuration property for more details. Please be advised that this feature is experimental and subject to backward-incompatible changes in future releases.
+
+
+---
+
+* [HDFS-1312](https://issues.apache.org/jira/browse/HDFS-1312) | *Major* | **Re-balance disks within a Datanode**
+
+The Disk Balancer lets administrators rebalance data across multiple disks of a DataNode. It is useful to correct skewed data distribution often seen after adding or replacing disks. Disk Balancer can be enabled by setting dfs.disk.balancer.enabled to true in hdfs-site.xml. It can be invoked by running "hdfs diskbalancer". See the "HDFS Diskbalancer" section in the HDFS Commands guide for detailed usage.
+
+
+---
+
+* [HADOOP-13263](https://issues.apache.org/jira/browse/HADOOP-13263) | *Major* | **Reload cached groups in background after expiry**
+
+hadoop.security.groups.cache.background.reload can be set to true to enable background reload of expired groups cache entries. This setting can improve the performance of services that use Groups.java (e.g. the NameNode) when group lookups are slow. The setting is disabled by default.
+
+
+---
+
+* [HDFS-10440](https://issues.apache.org/jira/browse/HDFS-10440) | *Major* | **Improve DataNode web UI**
+
+DataNode Web UI has been improved with new HTML5 page, showing useful information.
+
+
+---
+
+* [HADOOP-13209](https://issues.apache.org/jira/browse/HADOOP-13209) | *Major* | **replace slaves with workers**
+
+The 'slaves' file has been deprecated in favor of the 'workers' file and, other than the deprecation warnings, all references to slavery have been removed from the source tree.
+
+
+---
+
+* [HDFS-6434](https://issues.apache.org/jira/browse/HDFS-6434) | *Minor* | **Default permission for creating file should be 644 for WebHdfs/HttpFS**
+
+The default permissions of files and directories created via WebHDFS and HttpFS are now 644 and 755 respectively. See HDFS-10488 for related discussion.
+
+
+---
+
+* [HADOOP-12864](https://issues.apache.org/jira/browse/HADOOP-12864) | *Blocker* | **Remove bin/rcc script**
+
+The rcc command has been removed. See HADOOP-12485 where unused Hadoop Streaming classes were removed.
+
+
+---
+
+* [HADOOP-12709](https://issues.apache.org/jira/browse/HADOOP-12709) | *Major* | **Cut s3:// from trunk**
+
+The s3 file system has been removed. The s3a file system should be used instead.
+
+
+---
+
+* [HADOOP-12064](https://issues.apache.org/jira/browse/HADOOP-12064) | *Blocker* | **[JDK8] Update guice version to 4.0**
+
+Upgrading following dependences:
+\* Guice from 3.0 to 4.0
+\* cglib from 2.2 to 3.2.0
+\* asm from 3.2 to 5.0.4
+
+
+---
+
+* [HDFS-10548](https://issues.apache.org/jira/browse/HDFS-10548) | *Major* | **Remove the long deprecated BlockReaderRemote**
+
+This removes the configuration property {{dfs.client.use.legacy.blockreader}}, since the legacy remote block reader class has been removed from the codebase.
+
+
+---
+
+* [YARN-2928](https://issues.apache.org/jira/browse/YARN-2928) | *Critical* | **YARN Timeline Service v.2: alpha 1**
+
+We are introducing an early preview (alpha 1) of a major revision of YARN Timeline Service: v.2. YARN Timeline Service v.2 addresses two major challenges: improving scalability and reliability of Timeline Service, and enhancing usability by introducing flows and aggregation.
+
+YARN Timeline Service v.2 alpha 1 is provided so that users and developers can test it and provide feedback and suggestions for making it a ready replacement for Timeline Service v.1.x. It should be used only in a test capacity. Most importantly, security is not enabled. Do not set up or use Timeline Service v.2 until security is implemented if security is a critical requirement.
+
+More details are available in the [YARN Timeline Service v.2](./hadoop-yarn/hadoop-yarn-site/TimelineServiceV2.html) documentation.
+
+
+---
+
+* [HADOOP-13382](https://issues.apache.org/jira/browse/HADOOP-13382) | *Major* | **remove unneeded commons-httpclient dependencies from POM files in Hadoop and sub-projects**
+
+Dependencies on commons-httpclien
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/22] hadoop git commit: HADOOP-13481. User documents for Aliyun
OSS FileSystem. Contributed by Genmao Yu.
Posted by sh...@apache.org.
HADOOP-13481. User documents for Aliyun OSS FileSystem. Contributed by Genmao Yu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e671a0f5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e671a0f5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e671a0f5
Branch: refs/heads/HADOOP-12756
Commit: e671a0f52b5488b8453e1a3258ea5e6477995648
Parents: d33e928
Author: Mingfei <mi...@intel.com>
Authored: Sun Aug 28 10:37:52 2016 +0800
Committer: Mingfei <mi...@intel.com>
Committed: Wed Sep 7 11:17:43 2016 +0800
----------------------------------------------------------------------
.../apache/hadoop/fs/aliyun/oss/Constants.java | 3 +-
.../site/markdown/tools/hadoop-aliyun/index.md | 299 +++++++++++++++++++
2 files changed, 300 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e671a0f5/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
index 243fdd4..e0c05ed 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -95,8 +95,7 @@ public final class Constants {
// Comma separated list of directories
public static final String BUFFER_DIR_KEY = "fs.oss.buffer.dir";
- // private | public-read | public-read-write | authenticated-read |
- // log-delivery-write | bucket-owner-read | bucket-owner-full-control
+ // private | public-read | public-read-write
public static final String CANNED_ACL_KEY = "fs.oss.acl.default";
public static final String CANNED_ACL_DEFAULT = "";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e671a0f5/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
new file mode 100644
index 0000000..4095e06
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
@@ -0,0 +1,299 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Hadoop-Aliyun module: Integration with Aliyun Web Services
+
+<!-- MACRO{toc|fromDepth=0|toDepth=5} -->
+
+## Overview
+
+The `hadoop-aliyun` module provides support for Aliyun integration with
+[Aliyun Object Storage Service (Aliyun OSS)](https://www.aliyun.com/product/oss).
+The generated JAR file, `hadoop-aliyun.jar` also declares a transitive
+dependency on all external artifacts which are needed for this support \u2014 enabling
+downstream applications to easily use this support.
+
+To make it part of Apache Hadoop's default classpath, simply make sure
+that HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-aliyun' in the list.
+
+### Features
+
+* Read and write data stored in Aliyun OSS.
+* Present a hierarchical file system view by implementing the standard Hadoop
+[`FileSystem`](../api/org/apache/hadoop/fs/FileSystem.html) interface.
+* Can act as a source of data in a MapReduce job, or a sink.
+
+### Warning #1: Object Stores are not filesystems.
+
+Aliyun OSS is an example of "an object store". In order to achieve scalability
+and especially high availability, Aliyun OSS has relaxed some of the constraints
+which classic "POSIX" filesystems promise.
+
+
+
+Specifically
+
+1. Atomic operations: `delete()` and `rename()` are implemented by recursive
+file-by-file operations. They take time at least proportional to the number of files,
+during which time partial updates may be visible. `delete()` and `rename()`
+can not guarantee atomicity. If the operations are interrupted, the filesystem
+is left in an intermediate state.
+2. File owner and group are persisted, but the permissions model is not enforced.
+Authorization occurs at the level of the entire Aliyun account via
+[Aliyun Resource Access Management (Aliyun RAM)](https://www.aliyun.com/product/ram).
+3. Directory last access time is not tracked.
+4. The append operation is not supported.
+
+### Warning #2: Directory last access time is not tracked,
+features of Hadoop relying on this can have unexpected behaviour. E.g. the
+AggregatedLogDeletionService of YARN will not remove the appropriate logfiles.
+
+### Warning #3: Your Aliyun credentials are valuable
+
+Your Aliyun credentials not only pay for services, they offer read and write
+access to the data. Anyone with the account can not only read your datasets
+\u2014they can delete them.
+
+Do not inadvertently share these credentials through means such as
+1. Checking in to SCM any configuration files containing the secrets.
+2. Logging them to a console, as they invariably end up being seen.
+3. Defining filesystem URIs with the credentials in the URL, such as
+`oss://accessKeyId:accessKeySecret@directory/file`. They will end up in
+logs and error messages.
+4. Including the secrets in bug reports.
+
+If you do any of these: change your credentials immediately!
+
+### Warning #4: The Aliyun OSS client provided by Aliyun E-MapReduce are different from this implementation
+
+Specifically: on Aliyun E-MapReduce, `oss://` is also supported but with
+a different implementation. If you are using Aliyun E-MapReduce,
+follow these instructions \u2014and be aware that all issues related to Aliyun
+OSS integration in E-MapReduce can only be addressed by Aliyun themselves:
+please raise your issues with them.
+
+## OSS
+
+### Authentication properties
+
+ <property>
+ <name>fs.oss.accessKeyId</name>
+ <description>Aliyun access key ID</description>
+ </property>
+
+ <property>
+ <name>fs.oss.accessKeySecret</name>
+ <description>Aliyun access key secret</description>
+ </property>
+
+ <property>
+ <name>fs.oss.credentials.provider</name>
+ <description>
+ Class name of a credentials provider that implements
+ com.aliyun.oss.common.auth.CredentialsProvider. Omit if using access/secret keys
+ or another authentication mechanism. The specified class must provide an
+ accessible constructor accepting java.net.URI and
+ org.apache.hadoop.conf.Configuration, or an accessible default constructor.
+ </description>
+ </property>
+
+### Other properties
+
+ <property>
+ <name>fs.oss.endpoint</name>
+ <description>Aliyun OSS endpoint to connect to. An up-to-date list is
+ provided in the Aliyun OSS Documentation.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.oss.proxy.host</name>
+ <description>Hostname of the (optinal) proxy server for Aliyun OSS connection</description>
+ </property>
+
+ <property>
+ <name>fs.oss.proxy.port</name>
+ <description>Proxy server port</description>
+ </property>
+
+ <property>
+ <name>fs.oss.proxy.username</name>
+ <description>Username for authenticating with proxy server</description>
+ </property>
+
+ <property>
+ <name>fs.oss.proxy.password</name>
+ <description>Password for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.proxy.domain</name>
+ <description>Domain for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.proxy.workstation</name>
+ <description>Workstation for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.attempts.maximum</name>
+ <value>20</value>
+ <description>How many times we should retry commands on transient errors.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.connection.establish.timeout</name>
+ <value>50000</value>
+ <description>Connection setup timeout in milliseconds.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.connection.timeout</name>
+ <value>200000</value>
+ <description>Socket connection timeout in milliseconds.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.paging.maximum</name>
+ <value>500</value>
+ <description>How many keys to request from Aliyun OSS when doing directory listings at a time.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.oss.multipart.upload.size</name>
+ <value>10485760</value>
+ <description>Size of each of multipart pieces in bytes.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.multipart.upload.threshold</name>
+ <value>20971520</value>
+ <description>Minimum size in bytes before we start a multipart uploads or copy.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.multipart.download.size</name>
+ <value>102400/value>
+ <description>Size in bytes in each request from ALiyun OSS.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.buffer.dir</name>
+ <description>Comma separated list of directories to buffer OSS data before uploading to Aliyun OSS</description>
+ </property>
+
+ <property>
+ <name>fs.oss.buffer.dir</name>
+ <description>Comma separated list of directories to buffer OSS data before uploading to Aliyun OSS</description>
+ </property>
+
+ <property>
+ <name>fs.oss.acl.default</name>
+ <value></vaule>
+ <description>Set a canned ACL for bucket. Value may be private, public-read, public-read-write.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.oss.server-side-encryption-algorithm</name>
+ <value></vaule>
+ <description>Specify a server-side encryption algorithm for oss: file system.
+ Unset by default, and the only other currently allowable value is AES256.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.oss.connection.maximum</name>
+ <value>32</value>
+ <description>Number of simultaneous connections to oss.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.connection.secure.enabled</name>
+ <value>true</value>
+ <description>Connect to oss over ssl or not, true by default.</description>
+ </property>
+
+## Testing the hadoop-aliyun Module
+
+To test `oss://` filesystem client, two files which pass in authentication
+details to the test runner are needed.
+
+1. `auth-keys.xml`
+2. `core-site.xml`
+
+Those two configuration files must be put into
+`hadoop-tools/hadoop-aliyun/src/test/resources`.
+
+### `core-site.xml`
+
+This file pre-exists and sources the configurations created in `auth-keys.xml`.
+
+For most cases, no modification is needed, unless a specific, non-default property
+needs to be set during the testing.
+
+### `auth-keys.xml`
+
+This file triggers the testing of Aliyun OSS module. Without this file,
+*none of the tests in this module will be executed*
+
+It contains the access key Id/secret and proxy information that are needed to
+connect to Aliyun OSS, and an OSS bucket URL should be also provided.
+
+1. `test.fs.oss.name` : the URL of the bucket for Aliyun OSS tests
+
+The contents of the bucket will be cleaned during the testing process, so
+do not use the bucket for any purpose other than testing.
+
+### Run Hadoop contract tests
+Create file `contract-test-options.xml` under `/test/resources`. If a
+specific file `fs.contract.test.fs.oss` test path is not defined, those
+tests will be skipped. Credentials are also needed to run any of those
+tests, they can be copied from `auth-keys.xml` or through direct
+XInclude inclusion. Here is an example of `contract-test-options.xml`:
+
+ <?xml version="1.0"?>
+ <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+ <configuration>
+
+ <include xmlns="http://www.w3.org/2001/XInclude"
+ href="auth-keys.xml"/>
+
+ <property>
+ <name>fs.contract.test.fs.oss</name>
+ <value>oss://spark-tests</value>
+ </property>
+
+ <property>
+ <name>fs.oss.impl</name>
+ <value>org.apache.hadoop.fs.aliyun.AliyunOSSFileSystem</value>
+ </property>
+
+ <property>
+ <name>fs.oss.endpoint</name>
+ <value>oss-cn-hangzhou.aliyuncs.com</value>
+ </property>
+
+ <property>
+ <name>fs.oss.buffer.dir</name>
+ <value>/tmp/oss</value>
+ </property>
+
+ <property>
+ <name>fs.oss.multipart.download.size</name>
+ <value>102400</value>
+ </property>
+ </configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/22] hadoop git commit: HDFS-10841. Remove duplicate or unused
variable in appendFile(). Contributed by Kihwal Lee.
Posted by sh...@apache.org.
HDFS-10841. Remove duplicate or unused variable in appendFile(). Contributed by Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6c0b754
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6c0b754
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6c0b754
Branch: refs/heads/HADOOP-12756
Commit: f6c0b7543f612de756ff0c03e9a2c6e33b496a36
Parents: 7c213c3
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Sep 6 11:48:35 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Sep 6 11:48:35 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c0b754/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index 5192352..6f898ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -84,12 +84,10 @@ final class FSDirAppendOp {
final LocatedBlock lb;
final FSDirectory fsd = fsn.getFSDirectory();
- final String src;
final INodesInPath iip;
fsd.writeLock();
try {
iip = fsd.resolvePathForWrite(pc, srcArg);
- src = iip.getPath();
// Verify that the destination does not exist as a directory already
final INode inode = iip.getLastINode();
final String path = iip.getPath();
@@ -111,7 +109,7 @@ final class FSDirAppendOp {
// not support appending file with striped blocks
if (file.isStriped()) {
throw new UnsupportedOperationException(
- "Cannot append to files with striped block " + src);
+ "Cannot append to files with striped block " + path);
}
BlockManager blockManager = fsd.getBlockManager();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/22] hadoop git commit: HDFS-6962. ACL inheritance conflicts with
umaskmode. Contributed by Chris Nauroth.
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLIWithPosixAclInheritance.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLIWithPosixAclInheritance.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLIWithPosixAclInheritance.xml
new file mode 100644
index 0000000..7e9ace1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLIWithPosixAclInheritance.xml
@@ -0,0 +1,1075 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="testConf.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <!-- Normal mode is test. To run just the commands and dump the output
+ to the log, set it to nocompare -->
+ <mode>test</mode>
+
+ <!-- Comparator types:
+ ExactComparator
+ SubstringComparator
+ RegexpComparator
+ TokenComparator
+ -->
+ <tests>
+ <!-- Tests for setfacl and getfacl-->
+ <test>
+ <description>getfacl: basic permissions</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>getfacl: basic permissions for directory</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Add an ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r-- /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Add multiple ACLs at once</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:users:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove an ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,user:charlie:r-x /file1</command>
+ <command>-fs NAMENODE -setfacl -x user:bob /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!bob)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Add default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:users:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Add minimal default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user::rwx /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default\:mask)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : try adding default ACL to file</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>setfacl: Invalid ACL: only directories may have a default ACL</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove one default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl -x default:user:charlie /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:users:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default:user:charlie).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove all default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl -k /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:users:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove all but base ACLs for a directory</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl -b /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!charlie).*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default).*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!admin).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove all but base ACLs for a file</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /file1</command>
+ <command>-fs NAMENODE -setfacl -b /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!charlie).*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!admin).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : check inherit default ACL to file</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -touchz /dir1/file</command>
+ <command>-fs NAMENODE -getfacl /dir1/file</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1/file</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^mask::rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : check inherit default ACL to dir</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -mkdir /dir1/dir2</command>
+ <command>-fs NAMENODE -getfacl /dir1/dir2</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1/dir2</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^mask::rwx$</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>getfacl -R : recursive</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -mkdir /dir1/dir2</command>
+ <command>-fs NAMENODE -setfacl -m user:user1:r-x,group:users:rwx /dir1/dir2</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:r-x#LF#group::r-x#LF#group:admin:rwx#LF#mask::rwx#LF#other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:user1:r-x#LF#group::r-x#LF#group:users:rwx#LF#mask::rwx#LF#other::r-x#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl -R : recursive</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -mkdir /dir1/dir2</command>
+ <command>-fs NAMENODE -setfacl -R -m user:charlie:r-x,group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:r-x#LF#group::r-x#LF#group:admin:rwx#LF#mask::rwx#LF#other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:r-x#LF#group::r-x#LF#group:admin:rwx#LF#mask::rwx#LF#other::r-x#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl --set : Set full set of ACLs</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl --set user::rw-,group::r--,other::r--,user:user1:r-x,group:users:rw- /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:user1:r-x#LF#group::r--#LF#group:users:rw-#LF#mask::rwx#LF#other::r--#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl -x mask : remove mask entry along with other ACL entries</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl -x mask::,user:charlie,group:admin /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#group::r-x#LF#other::r-x#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>getfacl: only default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:rwx /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user:charlie:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>getfacl: effective permissions</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:rwx,group::-wx,group:sales:rwx,mask::r-x,default:user:charlie:rwx,default:group::r-x,default:group:sales:rwx,default:mask::rw- /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^user:charlie:rwx\t#effective:r-x$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^group::-wx\t#effective:--x$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^group:sales:rwx\t#effective:r-x$</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^default:user:charlie:rwx\t#effective:rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^default:group::r-x\t#effective:r--$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^default:group:sales:rwx\t#effective:rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>ls: display extended acl marker</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /dir1/dir2</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:rwx,group::-wx,group:sales:rwx,mask::r-x,default:user:charlie:rwx,default:group::r-x,default:group:sales:rwx,default:mask::rw- /dir1/dir2</command>
+ <command>-fs NAMENODE -ls /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^drwxr-xr-x\+( )*-( )*USERNAME( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir1/dir2</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl: recursive modify entries with mix of files and directories</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /dir1</command>
+ <command>-fs NAMENODE -touchz /dir1/file1</command>
+ <command>-fs NAMENODE -mkdir -p /dir1/dir2</command>
+ <command>-fs NAMENODE -touchz /dir1/dir2/file2</command>
+ <command>-fs NAMENODE -setfacl -R -m user:charlie:rwx,default:user:charlie:r-x /dir1</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2/file2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:charlie:rwx#LF#group::r--#LF#mask::rwx#LF#other::r--#LF##LF## file: /dir1/file1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:charlie:rwx#LF#group::r--#LF#mask::rwx#LF#other::r--#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl: recursive remove entries with mix of files and directories</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /dir1</command>
+ <command>-fs NAMENODE -touchz /dir1/file1</command>
+ <command>-fs NAMENODE -mkdir -p /dir1/dir2</command>
+ <command>-fs NAMENODE -touchz /dir1/dir2/file2</command>
+ <command>-fs NAMENODE -setfacl -R -m user:bob:rwx,user:charlie:rwx,default:user:bob:rwx,default:user:charlie:r-x /dir1</command>
+ <command>-fs NAMENODE -setfacl -R -x user:bob,default:user:bob /dir1</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2/file2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:charlie:rwx#LF#group::r--#LF#mask::rwx#LF#other::r--#LF##LF## file: /dir1/file1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:charlie:rwx#LF#group::r--#LF#mask::rwx#LF#other::r--#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl: recursive set with mix of files and directories</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /dir1</command>
+ <command>-fs NAMENODE -touchz /dir1/file1</command>
+ <command>-fs NAMENODE -mkdir -p /dir1/dir2</command>
+ <command>-fs NAMENODE -touchz /dir1/dir2/file2</command>
+ <command>-fs NAMENODE -setfacl -R --set user::rwx,user:charlie:rwx,group::r-x,other::r-x,default:user:charlie:r-x /dir1</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2/file2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF##LF## file: /dir1/file1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>copyFromLocal: copying file into a directory with a default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:rwx /dir1</command>
+ <command>-fs NAMENODE -copyFromLocal CLITEST_DATA/data1k /dir1/data1k</command>
+ <command>-fs NAMENODE -getfacl /dir1/data1k</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^# file: /dir1/data1k$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^# owner: USERNAME$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^# group: supergroup$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^user::rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^user:charlie:rwx\t#effective:rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^group::r-x\t#effective:r--$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^mask::rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^other::r--$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ </tests>
+</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/22] hadoop git commit: HADOOP-12756. Incorporate Aliyun OSS file
system implementation. Contributed by Mingfei Shi and Lin Zhou
Posted by sh...@apache.org.
HADOOP-12756. Incorporate Aliyun OSS file system implementation. Contributed by Mingfei Shi and Lin Zhou
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5d53422
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5d53422
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5d53422
Branch: refs/heads/HADOOP-12756
Commit: a5d5342228050a778b20e95adf7885bdba39985d
Parents: 5f23abf
Author: Kai Zheng <ka...@intel.com>
Authored: Thu Aug 4 21:21:10 2016 +0800
Committer: Mingfei <mi...@intel.com>
Committed: Wed Sep 7 11:15:47 2016 +0800
----------------------------------------------------------------------
.gitignore | 2 +
hadoop-project/pom.xml | 22 +
.../dev-support/findbugs-exclude.xml | 18 +
hadoop-tools/hadoop-aliyun/pom.xml | 133 +++
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 847 +++++++++++++++++++
.../fs/aliyun/oss/AliyunOSSInputStream.java | 268 ++++++
.../fs/aliyun/oss/AliyunOSSOutputStream.java | 219 +++++
.../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 151 ++++
.../apache/hadoop/fs/aliyun/oss/Constants.java | 110 +++
.../hadoop/fs/aliyun/oss/package-info.java | 22 +
.../hadoop/fs/aliyun/oss/OSSTestUtils.java | 80 ++
.../aliyun/oss/TestOSSFileSystemContract.java | 253 ++++++
.../fs/aliyun/oss/TestOSSInputStream.java | 141 +++
.../fs/aliyun/oss/TestOSSOutputStream.java | 71 ++
.../fs/aliyun/oss/contract/OSSContract.java | 54 ++
.../oss/contract/TestOSSContractCreate.java | 41 +
.../oss/contract/TestOSSContractDelete.java | 34 +
.../oss/contract/TestOSSContractMkdir.java | 34 +
.../oss/contract/TestOSSContractOpen.java | 34 +
.../oss/contract/TestOSSContractRename.java | 35 +
.../oss/contract/TestOSSContractSeek.java | 34 +
.../src/test/resources/contract/oss.xml | 105 +++
.../src/test/resources/core-site.xml | 46 +
.../src/test/resources/log4j.properties | 23 +
hadoop-tools/hadoop-tools-dist/pom.xml | 6 +
hadoop-tools/pom.xml | 1 +
26 files changed, 2784 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index a5d69d0..194862b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,5 @@ hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml
hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml
hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml
patchprocess/
+hadoop-tools/hadoop-aliyun/src/test/resources/auth-keys.xml
+hadoop-tools/hadoop-aliyun/src/test/resources/contract-test-options.xml
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 65e9672..29c2760 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -438,6 +438,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aliyun</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-kms</artifactId>
<version>${project.version}</version>
<classifier>classes</classifier>
@@ -1004,6 +1010,22 @@
<version>4.2.0</version>
</dependency>
+ <dependency>
+ <groupId>com.aliyun.oss</groupId>
+ <artifactId>aliyun-sdk-oss</artifactId>
+ <version>2.2.1</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml
new file mode 100644
index 0000000..40d78d0
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml
@@ -0,0 +1,18 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml
new file mode 100644
index 0000000..c87d13f
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/pom.xml
@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <relativePath>../../hadoop-project</relativePath>
+ </parent>
+ <artifactId>hadoop-aliyun</artifactId>
+ <name>Apache Hadoop Aliyun OSS support</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <file.encoding>UTF-8</file.encoding>
+ <downloadSources>true</downloadSources>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>tests-off</id>
+ <activation>
+ <file>
+ <missing>src/test/resources/auth-keys.xml</missing>
+ </file>
+ </activation>
+ <properties>
+ <maven.test.skip>true</maven.test.skip>
+ </properties>
+ </profile>
+ <profile>
+ <id>tests-on</id>
+ <activation>
+ <file>
+ <exists>src/test/resources/auth-keys.xml</exists>
+ </file>
+ </activation>
+ <properties>
+ <maven.test.skip>false</maven.test.skip>
+ </properties>
+ </profile>
+ </profiles>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <findbugsXmlOutput>true</findbugsXmlOutput>
+ <xmlOutput>true</xmlOutput>
+ <excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
+ </excludeFilterFile>
+ <effort>Max</effort>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <configuration>
+ <dependencyDetailsEnabled>false</dependencyDetailsEnabled>
+ <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>deplist</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>list</goal>
+ </goals>
+ <configuration>
+ <!-- build a shellprofile -->
+ <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.aliyun.oss</groupId>
+ <artifactId>aliyun-sdk-oss</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
new file mode 100644
index 0000000..30ddf8c
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -0,0 +1,847 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.aliyun.oss.common.auth.DefaultCredentialProvider;
+import com.aliyun.oss.common.auth.DefaultCredentials;
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSUtils.UserInfo;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.util.Progressable;
+
+import com.aliyun.oss.ClientConfiguration;
+import com.aliyun.oss.OSSClient;
+import com.aliyun.oss.OSSException;
+import com.aliyun.oss.common.comm.Protocol;
+import com.aliyun.oss.model.AbortMultipartUploadRequest;
+import com.aliyun.oss.model.CannedAccessControlList;
+import com.aliyun.oss.model.CompleteMultipartUploadRequest;
+import com.aliyun.oss.model.CompleteMultipartUploadResult;
+import com.aliyun.oss.model.CopyObjectResult;
+import com.aliyun.oss.model.DeleteObjectsRequest;
+import com.aliyun.oss.model.InitiateMultipartUploadRequest;
+import com.aliyun.oss.model.InitiateMultipartUploadResult;
+import com.aliyun.oss.model.ListObjectsRequest;
+import com.aliyun.oss.model.OSSObjectSummary;
+import com.aliyun.oss.model.ObjectListing;
+import com.aliyun.oss.model.ObjectMetadata;
+import com.aliyun.oss.model.PartETag;
+import com.aliyun.oss.model.UploadPartCopyRequest;
+import com.aliyun.oss.model.UploadPartCopyResult;
+
+/**
+ * Implementation of {@link FileSystem} for <a href="https://oss.aliyun.com">
+ * Aliyun OSS</a>, used to access OSS blob system in a filesystem style.
+ */
+public class AliyunOSSFileSystem extends FileSystem {
+
+ private URI uri;
+ private Path workingDir;
+ private OSSClient ossClient;
+ private String bucketName;
+ private long uploadPartSize;
+ private long multipartThreshold;
+ private int maxKeys;
+ private String serverSideEncryptionAlgorithm;
+
+ @Override
+ public FSDataOutputStream append(Path path, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new IOException("Append is not supported!");
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ if (ossClient != null) {
+ ossClient.shutdown();
+ }
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public FSDataOutputStream create(Path path, FsPermission permission,
+ boolean overwrite, int bufferSize, short replication, long blockSize,
+ Progressable progress) throws IOException {
+ String key = pathToKey(path);
+
+ if (!overwrite && exists(path)) {
+ throw new FileAlreadyExistsException(path + " already exists");
+ }
+
+ return new FSDataOutputStream(new AliyunOSSOutputStream(getConf(),
+ ossClient, bucketName, key, progress, statistics,
+ serverSideEncryptionAlgorithm), (Statistics)(null));
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ FileStatus status;
+ try {
+ status = getFileStatus(path);
+ } catch (FileNotFoundException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Couldn't delete " + path + ": Does not exist!");
+ }
+ return false;
+ }
+
+ String key = pathToKey(status.getPath());
+ if (status.isDirectory()) {
+ if (!key.endsWith("/")) {
+ key += "/";
+ }
+ if (!recursive) {
+ FileStatus[] statuses = listStatus(status.getPath());
+ // Check whether it is an empty directory or not
+ if (statuses.length > 0) {
+ throw new IOException("Cannot remove directory" + path +
+ ": It is not empty!");
+ } else {
+ // Delete empty directory without '-r'
+ ossClient.deleteObject(bucketName, key);
+ statistics.incrementWriteOps(1);
+ }
+ } else {
+ ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
+ listRequest.setPrefix(key);
+ listRequest.setMaxKeys(maxKeys);
+
+ while (true) {
+ ObjectListing objects = ossClient.listObjects(listRequest);
+ statistics.incrementReadOps(1);
+ List<String> keysToDelete = new ArrayList<String>();
+ for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
+ keysToDelete.add(objectSummary.getKey());
+ }
+ DeleteObjectsRequest deleteRequest =
+ new DeleteObjectsRequest(bucketName);
+ deleteRequest.setKeys(keysToDelete);
+ ossClient.deleteObjects(deleteRequest);
+ statistics.incrementWriteOps(1);
+ if (objects.isTruncated()) {
+ listRequest.setMarker(objects.getNextMarker());
+ } else {
+ break;
+ }
+ }
+ }
+ } else {
+ ossClient.deleteObject(bucketName, key);
+ statistics.incrementWriteOps(1);
+ }
+ //TODO: optimize logic here
+ try {
+ Path pPath = status.getPath().getParent();
+ FileStatus pStatus = getFileStatus(pPath);
+ if (pStatus.isDirectory()) {
+ return true;
+ } else {
+ throw new IOException("Path " + pPath +
+ " is assumed to be a directory!");
+ }
+ } catch (FileNotFoundException fnfe) {
+ // Make sure the parent directory exists
+ return mkdir(bucketName, pathToKey(status.getPath().getParent()));
+ }
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path path) throws IOException {
+ Path qualifiedPath = path.makeQualified(uri, workingDir);
+ String key = pathToKey(qualifiedPath);
+
+ // Root always exists
+ if (key.length() == 0) {
+ return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
+ }
+
+ ObjectMetadata meta = getObjectMetadata(key);
+ // If key not found and key does not end with "/"
+ if (meta == null && !key.endsWith("/")) {
+ // Case: dir + "/"
+ key += "/";
+ meta = getObjectMetadata(key);
+ }
+ if (meta == null) {
+ // Case: dir + "/" + file
+ ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
+ listRequest.setPrefix(key);
+ listRequest.setDelimiter("/");
+ listRequest.setMaxKeys(1);
+
+ ObjectListing listing = ossClient.listObjects(listRequest);
+ statistics.incrementReadOps(1);
+ if (!listing.getObjectSummaries().isEmpty() ||
+ !listing.getCommonPrefixes().isEmpty()) {
+ return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
+ } else {
+ throw new FileNotFoundException(path + ": No such file or directory!");
+ }
+ } else if (objectRepresentsDirectory(key, meta.getContentLength())) {
+ return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
+ } else {
+ return new FileStatus(meta.getContentLength(), false, 1,
+ getDefaultBlockSize(path), meta.getLastModified().getTime(),
+ qualifiedPath);
+ }
+ }
+
+ /**
+ * Return object metadata given object key.
+ *
+ * @param key object key
+ * @return return null if key does not exist
+ */
+ private ObjectMetadata getObjectMetadata(String key) {
+ try {
+ return ossClient.getObjectMetadata(bucketName, key);
+ } catch (OSSException osse) {
+ return null;
+ } finally {
+ statistics.incrementReadOps(1);
+ }
+ }
+
+ @Override
+ public String getScheme() {
+ return "oss";
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ @Deprecated
+ public long getDefaultBlockSize() {
+ return getConf().getLong(FS_OSS_BLOCK_SIZE_KEY, FS_OSS_BLOCK_SIZE_DEFAULT);
+ }
+
+ @Override
+ public String getCanonicalServiceName() {
+ // Does not support Token
+ return null;
+ }
+
+ /**
+ * Initialize new FileSystem.
+ *
+ * @param name the uri of the file system, including host, port, etc.
+ *
+ * @param conf configuration of the file system
+ * @throws IOException IO problems
+ */
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+
+ uri = java.net.URI.create(name.getScheme() + "://" + name.getAuthority());
+ workingDir =
+ new Path("/user",
+ System.getProperty("user.name")).makeQualified(uri, null);
+
+ bucketName = name.getHost();
+
+ ClientConfiguration clientConf = new ClientConfiguration();
+ clientConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS_KEY,
+ MAXIMUM_CONNECTIONS_DEFAULT));
+ boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS_KEY,
+ SECURE_CONNECTIONS_DEFAULT);
+ clientConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
+ clientConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES_KEY,
+ MAX_ERROR_RETRIES_DEFAULT));
+ clientConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT_KEY,
+ ESTABLISH_TIMEOUT_DEFAULT));
+ clientConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT_KEY,
+ SOCKET_TIMEOUT_DEFAULT));
+
+ String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, "");
+ int proxyPort = conf.getInt(PROXY_PORT_KEY, -1);
+ if (!proxyHost.isEmpty()) {
+ clientConf.setProxyHost(proxyHost);
+ if (proxyPort >= 0) {
+ clientConf.setProxyPort(proxyPort);
+ } else {
+ if (secureConnections) {
+ LOG.warn("Proxy host set without port. Using HTTPS default 443");
+ clientConf.setProxyPort(443);
+ } else {
+ LOG.warn("Proxy host set without port. Using HTTP default 80");
+ clientConf.setProxyPort(80);
+ }
+ }
+ String proxyUsername = conf.getTrimmed(PROXY_USERNAME_KEY);
+ String proxyPassword = conf.getTrimmed(PROXY_PASSWORD_KEY);
+ if ((proxyUsername == null) != (proxyPassword == null)) {
+ String msg = "Proxy error: " + PROXY_USERNAME_KEY + " or " +
+ PROXY_PASSWORD_KEY + " set without the other.";
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+ clientConf.setProxyUsername(proxyUsername);
+ clientConf.setProxyPassword(proxyPassword);
+ clientConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN_KEY));
+ clientConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION_KEY));
+ } else if (proxyPort >= 0) {
+ String msg = "Proxy error: " + PROXY_PORT_KEY + " set without " +
+ PROXY_HOST_KEY;
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+
+ String endPoint = conf.getTrimmed(ENDPOINT_KEY, "");
+ ossClient =
+ new OSSClient(endPoint, getCredentialsProvider(name, conf), clientConf);
+
+ maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
+ uploadPartSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
+ MULTIPART_UPLOAD_SIZE_DEFAULT);
+ multipartThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
+ MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
+
+ if (uploadPartSize < 5 * 1024 * 1024) {
+ LOG.warn(MULTIPART_UPLOAD_SIZE_KEY + " must be at least 5 MB");
+ uploadPartSize = 5 * 1024 * 1024;
+ }
+
+ if (multipartThreshold < 5 * 1024 * 1024) {
+ LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be at least 5 MB");
+ multipartThreshold = 5 * 1024 * 1024;
+ }
+
+ if (multipartThreshold > 1024 * 1024 * 1024) {
+ LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be less than 1 GB");
+ multipartThreshold = 1024 * 1024 * 1024;
+ }
+
+ String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT);
+ if (!cannedACLName.isEmpty()) {
+ CannedAccessControlList cannedACL =
+ CannedAccessControlList.valueOf(cannedACLName);
+ ossClient.setBucketAcl(bucketName, cannedACL);
+ }
+
+ serverSideEncryptionAlgorithm =
+ conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY, "");
+
+ setConf(conf);
+ }
+
+ /**
+ * Create the default credential provider, or load in one explicitly
+ * identified in the configuration.
+ * @param name the uri of the file system
+ * @param conf configuration
+ * @return a credential provider
+ * @throws IOException on any problem. Class construction issues may be
+ * nested inside the IOE.
+ */
+ private CredentialsProvider getCredentialsProvider(URI name,
+ Configuration conf) throws IOException {
+ CredentialsProvider credentials;
+
+ String className = conf.getTrimmed(ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY);
+ if (StringUtils.isEmpty(className)) {
+ Configuration newConf =
+ ProviderUtils.excludeIncompatibleCredentialProviders(conf,
+ AliyunOSSFileSystem.class);
+ String accessKey =
+ AliyunOSSUtils.getPassword(newConf, ACCESS_KEY,
+ UserInfo.EMPTY.getUser());
+ String secretKey =
+ AliyunOSSUtils.getPassword(newConf, SECRET_KEY,
+ UserInfo.EMPTY.getPassword());
+ credentials =
+ new DefaultCredentialProvider(
+ new DefaultCredentials(accessKey, secretKey));
+
+ } else {
+ try {
+ LOG.debug("Credential provider class is:" + className);
+ Class<?> credClass = Class.forName(className);
+ try {
+ credentials =
+ (CredentialsProvider)credClass.getDeclaredConstructor(
+ URI.class, Configuration.class).newInstance(this.uri, conf);
+ } catch (NoSuchMethodException | SecurityException e) {
+ credentials =
+ (CredentialsProvider)credClass.getDeclaredConstructor()
+ .newInstance();
+ }
+ } catch (ClassNotFoundException e) {
+ throw new IOException(className + " not found.", e);
+ } catch (NoSuchMethodException | SecurityException e) {
+ throw new IOException(String.format("%s constructor exception. A " +
+ "class specified in %s must provide an accessible constructor " +
+ "accepting URI and Configuration, or an accessible default " +
+ "constructor.", className, ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY), e);
+ } catch (ReflectiveOperationException | IllegalArgumentException e) {
+ throw new IOException(className + " instantiation exception.", e);
+ }
+ }
+
+ return credentials;
+ }
+
+ /**
+ * Check if OSS object represents a directory.
+ *
+ * @param name object key
+ * @param size object content length
+ * @return true if object represents a directory
+ */
+ private boolean objectRepresentsDirectory(final String name,
+ final long size) {
+ return !name.isEmpty() && name.endsWith("/") && size == 0L;
+ }
+
+ /**
+ * Turns a path (relative or otherwise) into an OSS key.
+ *
+ * @param path the path of the file
+ * @return the key of the object that represent the file
+ */
+ private String pathToKey(Path path) {
+ if (!path.isAbsolute()) {
+ path = new Path(workingDir, path);
+ }
+
+ if (path.toUri().getScheme() != null && path.toUri().getPath().isEmpty()) {
+ return "";
+ }
+
+ return path.toUri().getPath().substring(1);
+ }
+
+ private Path keyToPath(String key) {
+ return new Path("/" + key);
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ String key = pathToKey(path);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("List status for path: " + path);
+ }
+
+ final List<FileStatus> result = new ArrayList<FileStatus>();
+ final FileStatus fileStatus = getFileStatus(path);
+
+ if (fileStatus.isDirectory()) {
+ if (!key.endsWith("/")) {
+ key = key + "/";
+ }
+
+ ListObjectsRequest listObjectsRequest =
+ new ListObjectsRequest(bucketName);
+ listObjectsRequest.setPrefix(key);
+ listObjectsRequest.setDelimiter("/");
+ listObjectsRequest.setMaxKeys(maxKeys);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("listStatus: doing listObjects for directory " + key);
+ }
+
+ while (true) {
+ ObjectListing objects = ossClient.listObjects(listObjectsRequest);
+ statistics.incrementReadOps(1);
+ for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
+ Path keyPath = keyToPath(objectSummary.getKey())
+ .makeQualified(uri, workingDir);
+ if (keyPath.equals(path)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Ignoring: " + keyPath);
+ }
+ continue;
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding: fi: " + keyPath);
+ }
+ result.add(new FileStatus(objectSummary.getSize(), false, 1,
+ getDefaultBlockSize(keyPath),
+ objectSummary.getLastModified().getTime(), keyPath));
+ }
+ }
+
+ for (String prefix : objects.getCommonPrefixes()) {
+ Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir);
+ if (keyPath.equals(path)) {
+ continue;
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding: rd: " + keyPath);
+ }
+ result.add(new FileStatus(0, true, 1, 0, 0, keyPath));
+ }
+ }
+
+ if (objects.isTruncated()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("listStatus: list truncated - getting next batch");
+ }
+ listObjectsRequest.setMarker(objects.getNextMarker());
+ statistics.incrementReadOps(1);
+ } else {
+ break;
+ }
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding: rd (not a dir): " + path);
+ }
+ result.add(fileStatus);
+ }
+
+ return result.toArray(new FileStatus[result.size()]);
+ }
+
+ /**
+ * Used to create an empty file that represents an empty directory.
+ *
+ * @param bucketName the bucket this directory belongs to
+ * @param objectName directory path
+ * @return true if directory successfully created
+ * @throws IOException
+ */
+ private boolean mkdir(final String bucket, final String objectName)
+ throws IOException {
+ String dirName = objectName;
+ ObjectMetadata dirMeta = new ObjectMetadata();
+ byte[] buffer = new byte[0];
+ ByteArrayInputStream in = new ByteArrayInputStream(buffer);
+ dirMeta.setContentLength(0);
+ if (!objectName.endsWith("/")) {
+ dirName += "/";
+ }
+ try {
+ ossClient.putObject(bucket, dirName, in, dirMeta);
+ return true;
+ } finally {
+ in.close();
+ }
+ }
+
+ @Override
+ public boolean mkdirs(Path path, FsPermission permission)
+ throws IOException {
+ try {
+ FileStatus fileStatus = getFileStatus(path);
+
+ if (fileStatus.isDirectory()) {
+ return true;
+ } else {
+ throw new FileAlreadyExistsException("Path is a file: " + path);
+ }
+ } catch (FileNotFoundException e) {
+ validatePath(path);
+ String key = pathToKey(path);
+ return mkdir(bucketName, key);
+ }
+ }
+
+ /**
+ * Check whether the path is a valid path.
+ *
+ * @param path the path to be checked
+ * @throws IOException
+ */
+ private void validatePath(Path path) throws IOException {
+ Path fPart = path.getParent();
+ do {
+ try {
+ FileStatus fileStatus = getFileStatus(fPart);
+ if (fileStatus.isDirectory()) {
+ // If path exists and a directory, exit
+ break;
+ } else {
+ throw new FileAlreadyExistsException(String.format(
+ "Can't make directory for path '%s', it is a file.", fPart));
+ }
+ } catch (FileNotFoundException fnfe) {
+ }
+ fPart = fPart.getParent();
+ } while (fPart != null);
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ final FileStatus fileStatus = getFileStatus(path);
+ if (fileStatus.isDirectory()) {
+ throw new FileNotFoundException("Can't open " + path +
+ " because it is a directory");
+ }
+
+ return new FSDataInputStream(new AliyunOSSInputStream(getConf(), ossClient,
+ bucketName, pathToKey(path), fileStatus.getLen(), statistics));
+ }
+
+ @Override
+ public boolean rename(Path srcPath, Path dstPath) throws IOException {
+ if (srcPath.isRoot()) {
+ // Cannot rename root of file system
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cannot rename the root of a filesystem");
+ }
+ return false;
+ }
+ Path parent = dstPath.getParent();
+ while (parent != null && !srcPath.equals(parent)) {
+ parent = parent.getParent();
+ }
+ if (parent != null) {
+ return false;
+ }
+ FileStatus srcStatus = getFileStatus(srcPath);
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dstPath);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst dir exists or not
+ dstStatus = getFileStatus(dstPath.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", srcPath, dstPath,
+ dstPath.getParent()));
+ }
+ } else {
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory
+ dstPath = new Path(dstPath, srcPath.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dstPath);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory / not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ srcPath, dstPath));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", srcPath,
+ dstPath));
+ }
+ }
+ if (srcStatus.isDirectory()) {
+ copyDirectory(srcPath, dstPath);
+ } else {
+ copyFile(srcPath, dstPath);
+ }
+ if (srcPath.equals(dstPath)) {
+ return true;
+ } else {
+ return delete(srcPath, true);
+ }
+ }
+
+ /**
+ * Copy file from source path to destination path.
+ * (the caller should make sure srcPath is a file and dstPath is valid.)
+ *
+ * @param srcPath source path
+ * @param dstPath destination path
+ * @return true if successfully copied
+ */
+ private boolean copyFile(Path srcPath, Path dstPath) {
+ String srcKey = pathToKey(srcPath);
+ String dstKey = pathToKey(dstPath);
+ return copyFile(srcKey, dstKey);
+ }
+
+ /**
+ * Copy an object from source key to destination key.
+ *
+ * @param srcKey source key
+ * @param dstKey destination key
+ * @return true if successfully copied
+ */
+ private boolean copyFile(String srcKey, String dstKey) {
+ ObjectMetadata objectMeta =
+ ossClient.getObjectMetadata(bucketName, srcKey);
+ long dataLen = objectMeta.getContentLength();
+ if (dataLen <= multipartThreshold) {
+ return singleCopy(srcKey, dstKey);
+ } else {
+ return multipartCopy(srcKey, dataLen, dstKey);
+ }
+ }
+
+ /**
+ * Use single copy to copy an oss object.
+ *
+ * @param srcKey source key
+ * @param dstKey destination key
+ * @return true if successfully copied
+ * (the caller should make sure srcPath is a file and dstPath is valid)
+ */
+ private boolean singleCopy(String srcKey, String dstKey) {
+ CopyObjectResult copyResult =
+ ossClient.copyObject(bucketName, srcKey, bucketName, dstKey);
+ LOG.debug(copyResult.getETag());
+ return true;
+ }
+
+ /**
+ * Use multipart copy to copy an oss object.
+ * (the caller should make sure srcPath is a file and dstPath is valid)
+ *
+ * @param srcKey source key
+ * @param dataLen data size of the object to copy
+ * @param dstKey destination key
+ * @return true if successfully copied, or false if upload is aborted
+ */
+ private boolean multipartCopy(String srcKey, long dataLen, String dstKey) {
+ int partNum = (int)(dataLen / uploadPartSize);
+ if (dataLen % uploadPartSize != 0) {
+ partNum++;
+ }
+ InitiateMultipartUploadRequest initiateMultipartUploadRequest =
+ new InitiateMultipartUploadRequest(bucketName, dstKey);
+ ObjectMetadata meta = new ObjectMetadata();
+ if (!serverSideEncryptionAlgorithm.isEmpty()) {
+ meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
+ }
+ initiateMultipartUploadRequest.setObjectMetadata(meta);
+ InitiateMultipartUploadResult initiateMultipartUploadResult =
+ ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
+ String uploadId = initiateMultipartUploadResult.getUploadId();
+ List<PartETag> partETags = new ArrayList<PartETag>();
+ try {
+ for (int i = 0; i < partNum; i++) {
+ long skipBytes = uploadPartSize * i;
+ long size = (uploadPartSize < dataLen - skipBytes) ?
+ uploadPartSize : dataLen - skipBytes;
+ UploadPartCopyRequest partCopyRequest = new UploadPartCopyRequest();
+ partCopyRequest.setSourceBucketName(bucketName);
+ partCopyRequest.setSourceKey(srcKey);
+ partCopyRequest.setBucketName(bucketName);
+ partCopyRequest.setKey(dstKey);
+ partCopyRequest.setUploadId(uploadId);
+ partCopyRequest.setPartSize(size);
+ partCopyRequest.setBeginIndex(skipBytes);
+ partCopyRequest.setPartNumber(i + 1);
+ UploadPartCopyResult partCopyResult =
+ ossClient.uploadPartCopy(partCopyRequest);
+ statistics.incrementWriteOps(1);
+ partETags.add(partCopyResult.getPartETag());
+ }
+ CompleteMultipartUploadRequest completeMultipartUploadRequest =
+ new CompleteMultipartUploadRequest(bucketName, dstKey,
+ uploadId, partETags);
+ CompleteMultipartUploadResult completeMultipartUploadResult =
+ ossClient.completeMultipartUpload(completeMultipartUploadRequest);
+ LOG.debug(completeMultipartUploadResult.getETag());
+ return true;
+ } catch (Exception e) {
+ AbortMultipartUploadRequest abortMultipartUploadRequest =
+ new AbortMultipartUploadRequest(bucketName, dstKey, uploadId);
+ ossClient.abortMultipartUpload(abortMultipartUploadRequest);
+ return false;
+ }
+ }
+
+ /**
+ * Copy a directory from source path to destination path.
+ * (the caller should make sure srcPath is a directory, and dstPath is valid)
+ *
+ * @param srcPath source path
+ * @param dstPath destination path
+ * @return true if successfully copied
+ */
+ private boolean copyDirectory(Path srcPath, Path dstPath) {
+ String srcKey = pathToKey(srcPath);
+ String dstKey = pathToKey(dstPath);
+
+ if (!srcKey.endsWith("/")) {
+ srcKey = srcKey + "/";
+ }
+ if (!dstKey.endsWith("/")) {
+ dstKey = dstKey + "/";
+ }
+
+ if (dstKey.startsWith(srcKey)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cannot rename a directory to a subdirectory of self");
+ }
+ return false;
+ }
+
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName);
+ listObjectsRequest.setPrefix(srcKey);
+ listObjectsRequest.setMaxKeys(maxKeys);
+
+ ObjectListing objects = ossClient.listObjects(listObjectsRequest);
+ statistics.incrementReadOps(1);
+ // Copy files from src folder to dst
+ while (true) {
+ for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
+ String newKey =
+ dstKey.concat(objectSummary.getKey().substring(srcKey.length()));
+ copyFile(objectSummary.getKey(), newKey);
+ }
+ if (objects.isTruncated()) {
+ listObjectsRequest.setMarker(objects.getNextMarker());
+ statistics.incrementReadOps(1);
+ } else {
+ break;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public void setWorkingDirectory(Path dir) {
+ this.workingDir = dir;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
new file mode 100644
index 0000000..bcd00dc
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem.Statistics;
+
+import com.aliyun.oss.OSSClient;
+import com.aliyun.oss.model.GetObjectRequest;
+
+/**
+ * The input stream for OSS blob system.
+ * The class uses multi-part downloading to read data from the object content
+ * stream.
+ */
+public class AliyunOSSInputStream extends FSInputStream {
+ public static final Log LOG = LogFactory.getLog(AliyunOSSInputStream.class);
+ private static final int MAX_RETRIES = 10;
+ private final long downloadPartSize;
+
+ private String bucketName;
+ private String key;
+ private OSSClient ossClient;
+ private Statistics statistics;
+ private boolean closed;
+ private InputStream wrappedStream = null;
+ private long dataLen;
+ private long position;
+ private long partRemaining;
+
+ public AliyunOSSInputStream(Configuration conf, OSSClient client,
+ String bucketName, String key, Long dataLen, Statistics statistics)
+ throws IOException {
+ this.bucketName = bucketName;
+ this.key = key;
+ ossClient = client;
+ this.statistics = statistics;
+ this.dataLen = dataLen;
+ downloadPartSize = conf.getLong(MULTIPART_DOWNLOAD_SIZE_KEY,
+ MULTIPART_DOWNLOAD_SIZE_DEFAULT);
+ reopen(0);
+ closed = false;
+ }
+
+ /**
+ * Reopen the wrapped stream at give position, by seeking for
+ * data of a part length from object content stream.
+ *
+ * @param pos position from start of a file
+ * @throws IOException if failed to reopen
+ */
+ private synchronized void reopen(long pos) throws IOException {
+
+ long partLen;
+
+ if (pos < 0) {
+ throw new EOFException("Cannot seek at negtive position:" + pos);
+ } else if (pos > dataLen) {
+ throw new EOFException("Cannot seek after EOF, fileLen:" + dataLen +
+ " position:" + pos);
+ } else if (pos + downloadPartSize > dataLen) {
+ partLen = dataLen - pos;
+ } else {
+ partLen = downloadPartSize;
+ }
+
+ if (wrappedStream != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Aborting old stream to open at pos " + pos);
+ }
+ wrappedStream.close();
+ }
+
+ GetObjectRequest request = new GetObjectRequest(bucketName, key);
+ request.setRange(pos, pos + partLen - 1);
+ wrappedStream = ossClient.getObject(request).getObjectContent();
+ if (wrappedStream == null) {
+ throw new IOException("Null IO stream");
+ }
+ position = pos;
+ partRemaining = partLen;
+ }
+
+ @Override
+ public synchronized int read() throws IOException {
+ checkNotClosed();
+
+ if (partRemaining <= 0 && position < dataLen) {
+ reopen(position);
+ }
+
+ int tries = MAX_RETRIES;
+ boolean retry;
+ int byteRead = -1;
+ do {
+ retry = false;
+ try {
+ byteRead = wrappedStream.read();
+ } catch (Exception e) {
+ handleReadException(e, --tries);
+ retry = true;
+ }
+ } while (retry);
+ if (byteRead >= 0) {
+ position++;
+ partRemaining--;
+ }
+
+ if (statistics != null && byteRead >= 0) {
+ statistics.incrementBytesRead(1);
+ }
+ return byteRead;
+ }
+
+
+ /**
+ * Check whether the input stream is closed.
+ *
+ * @throws IOException if stream is closed
+ */
+ private void checkNotClosed() throws IOException {
+ if (closed) {
+ throw new IOException("Stream is closed!");
+ }
+ }
+
+ @Override
+ public synchronized int read(byte[] buf, int off, int len)
+ throws IOException {
+ checkNotClosed();
+
+ if (buf == null) {
+ throw new NullPointerException();
+ } else if (off < 0 || len < 0 || len > buf.length - off) {
+ throw new IndexOutOfBoundsException();
+ } else if (len == 0) {
+ return 0;
+ }
+
+ int bytesRead = 0;
+ // Not EOF, and read not done
+ while (position < dataLen && bytesRead < len) {
+ if (partRemaining == 0) {
+ reopen(position);
+ }
+
+ int tries = MAX_RETRIES;
+ boolean retry;
+ int bytes = -1;
+ do {
+ retry = false;
+ try {
+ bytes = wrappedStream.read(buf, off + bytesRead, len - bytesRead);
+ } catch (Exception e) {
+ handleReadException(e, --tries);
+ retry = true;
+ }
+ } while (retry);
+
+ if (bytes > 0) {
+ bytesRead += bytes;
+ position += bytes;
+ partRemaining -= bytes;
+ } else if (partRemaining != 0) {
+ throw new IOException("Failed to read from stream. Remaining:" +
+ partRemaining);
+ }
+ }
+
+ if (statistics != null && bytesRead > 0) {
+ statistics.incrementBytesRead(bytesRead);
+ }
+
+ // Read nothing, but attempt to read something
+ if (bytesRead == 0 && len > 0) {
+ return -1;
+ } else {
+ return bytesRead;
+ }
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ if (wrappedStream != null) {
+ wrappedStream.close();
+ }
+ }
+
+ @Override
+ public synchronized int available() throws IOException {
+ checkNotClosed();
+
+ long remaining = dataLen - position;
+ if (remaining > Integer.MAX_VALUE) {
+ return Integer.MAX_VALUE;
+ }
+ return (int)remaining;
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ checkNotClosed();
+ if (position == pos) {
+ return;
+ } else if (pos > position && pos < position + partRemaining) {
+ wrappedStream.skip(pos - position);
+ position = pos;
+ } else {
+ reopen(pos);
+ }
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ checkNotClosed();
+ return position;
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ checkNotClosed();
+ return false;
+ }
+
+ private void handleReadException(Exception e, int tries) throws IOException{
+ if (tries == 0) {
+ throw new IOException(e);
+ }
+
+ LOG.warn("Some exceptions occurred in oss connection, try to reopen oss" +
+ " connection at position '" + position + "', " + e.getMessage());
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e2) {
+ LOG.warn(e2.getMessage());
+ }
+ reopen(position);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
new file mode 100644
index 0000000..589e014
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.util.Progressable;
+
+import com.aliyun.oss.OSSClient;
+import com.aliyun.oss.model.AbortMultipartUploadRequest;
+import com.aliyun.oss.model.CompleteMultipartUploadRequest;
+import com.aliyun.oss.model.CompleteMultipartUploadResult;
+import com.aliyun.oss.model.InitiateMultipartUploadRequest;
+import com.aliyun.oss.model.InitiateMultipartUploadResult;
+import com.aliyun.oss.model.ObjectMetadata;
+import com.aliyun.oss.model.PartETag;
+import com.aliyun.oss.model.PutObjectResult;
+import com.aliyun.oss.model.UploadPartRequest;
+import com.aliyun.oss.model.UploadPartResult;
+
+/**
+ * The output stream for OSS blob system.
+ * Data will be buffered on local disk, then uploaded to OSS in
+ * {@link #close()} method.
+ */
+public class AliyunOSSOutputStream extends OutputStream {
+ public static final Log LOG = LogFactory.getLog(AliyunOSSOutputStream.class);
+ private String bucketName;
+ private String key;
+ private Statistics statistics;
+ private Progressable progress;
+ private String serverSideEncryptionAlgorithm;
+ private long partSize;
+ private long partSizeThreshold;
+ private LocalDirAllocator dirAlloc;
+ private boolean closed;
+ private File tmpFile;
+ private BufferedOutputStream backupStream;
+ private OSSClient ossClient;
+
+ public AliyunOSSOutputStream(Configuration conf, OSSClient client,
+ String bucketName, String key, Progressable progress,
+ Statistics statistics, String serverSideEncryptionAlgorithm)
+ throws IOException {
+ this.bucketName = bucketName;
+ this.key = key;
+ // The caller cann't get any progress information
+ this.progress = progress;
+ ossClient = client;
+ this.statistics = statistics;
+ this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm;
+
+ partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
+ MULTIPART_UPLOAD_SIZE_DEFAULT);
+ partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
+ MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
+
+ if (conf.get(BUFFER_DIR_KEY) == null) {
+ conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss");
+ }
+ dirAlloc = new LocalDirAllocator(BUFFER_DIR_KEY);
+
+ tmpFile = dirAlloc.createTmpFileForWrite("output-",
+ LocalDirAllocator.SIZE_UNKNOWN, conf);
+ backupStream = new BufferedOutputStream(new FileOutputStream(tmpFile));
+ closed = false;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ if (backupStream != null) {
+ backupStream.close();
+ }
+ long dataLen = tmpFile.length();
+ try {
+ if (dataLen <= partSizeThreshold) {
+ uploadObject();
+ } else {
+ multipartUploadObject();
+ }
+ } finally {
+ tmpFile.delete();
+ }
+ }
+
+ /**
+ * Upload temporary file as an OSS object, using single upload.
+ *
+ * @throws IOException
+ */
+ private void uploadObject() throws IOException {
+ File object = tmpFile.getAbsoluteFile();
+ FileInputStream fis = new FileInputStream(object);
+ ObjectMetadata meta = new ObjectMetadata();
+ meta.setContentLength(object.length());
+ if (!serverSideEncryptionAlgorithm.isEmpty()) {
+ meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
+ }
+ try {
+ PutObjectResult result = ossClient.putObject(bucketName, key, fis, meta);
+ LOG.debug(result.getETag());
+ statistics.incrementWriteOps(1);
+ } finally {
+ fis.close();
+ }
+ }
+
+ /**
+ * Upload temporary file as an OSS object, using multipart upload.
+ *
+ * @throws IOException
+ */
+ private void multipartUploadObject() throws IOException {
+ File object = tmpFile.getAbsoluteFile();
+ long dataLen = object.length();
+ InitiateMultipartUploadRequest initiateMultipartUploadRequest =
+ new InitiateMultipartUploadRequest(bucketName, key);
+ ObjectMetadata meta = new ObjectMetadata();
+ // meta.setContentLength(dataLen);
+ if (!serverSideEncryptionAlgorithm.isEmpty()) {
+ meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
+ }
+ initiateMultipartUploadRequest.setObjectMetadata(meta);
+ InitiateMultipartUploadResult initiateMultipartUploadResult =
+ ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
+ int partNum = (int)(dataLen / partSize);
+ if (dataLen % partSize != 0) {
+ partNum += 1;
+ }
+ if (partNum > MULTIPART_UPLOAD_PART_NUM_LIMIT) {
+ throw new IOException("Number of parts " + partNum + " should not be " +
+ "bigger than limit " + MULTIPART_UPLOAD_PART_NUM_LIMIT);
+ }
+ List<PartETag> partETags = new ArrayList<PartETag>();
+ String uploadId = initiateMultipartUploadResult.getUploadId();
+
+ try {
+ for (int i = 0; i < partNum; i++) {
+ // TODO: Optimize this, avoid opening the object multiple times
+ FileInputStream fis = new FileInputStream(object);
+ try {
+ long skipBytes = partSize * i;
+ fis.skip(skipBytes);
+ long size = (partSize < dataLen - skipBytes) ?
+ partSize : dataLen - skipBytes;
+ UploadPartRequest uploadPartRequest = new UploadPartRequest();
+ uploadPartRequest.setBucketName(bucketName);
+ uploadPartRequest.setKey(key);
+ uploadPartRequest.setUploadId(uploadId);
+ uploadPartRequest.setInputStream(fis);
+ uploadPartRequest.setPartSize(size);
+ uploadPartRequest.setPartNumber(i + 1);
+ UploadPartResult uploadPartResult =
+ ossClient.uploadPart(uploadPartRequest);
+ statistics.incrementWriteOps(1);
+ partETags.add(uploadPartResult.getPartETag());
+ } finally {
+ fis.close();
+ }
+ }
+ CompleteMultipartUploadRequest completeMultipartUploadRequest =
+ new CompleteMultipartUploadRequest(bucketName, key,
+ uploadId, partETags);
+ CompleteMultipartUploadResult completeMultipartUploadResult =
+ ossClient.completeMultipartUpload(completeMultipartUploadRequest);
+ LOG.debug(completeMultipartUploadResult.getETag());
+ } catch (Exception e) {
+ AbortMultipartUploadRequest abortMultipartUploadRequest =
+ new AbortMultipartUploadRequest(bucketName, key, uploadId);
+ ossClient.abortMultipartUpload(abortMultipartUploadRequest);
+ }
+ }
+
+ @Override
+ public synchronized void flush() throws IOException {
+ backupStream.flush();
+ }
+
+ @Override
+ public synchronized void write(int b) throws IOException {
+ backupStream.write(b);
+ statistics.incrementBytesWritten(1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
new file mode 100644
index 0000000..3f66a4f
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URLDecoder;
+import java.util.Objects;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Utility methods for Aliyun OSS code.
+ */
+final public class AliyunOSSUtils {
+ private AliyunOSSUtils() {
+ }
+
+ /**
+ * User information includes user name and password.
+ */
+ static public class UserInfo {
+ private final String user;
+ private final String password;
+
+ public static final UserInfo EMPTY = new UserInfo("", "");
+
+ public UserInfo(String user, String password) {
+ this.user = user;
+ this.password = password;
+ }
+
+ /**
+ * Predicate to verify user information is set.
+ * @return true if the username is defined (not null, not empty).
+ */
+ public boolean hasLogin() {
+ return StringUtils.isNotEmpty(user);
+ }
+
+ /**
+ * Equality test matches user and password.
+ * @param o other object
+ * @return true if the objects are considered equivalent.
+ */
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ UserInfo that = (UserInfo) o;
+ return Objects.equals(user, that.user) &&
+ Objects.equals(password, that.password);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(user, password);
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+ }
+
+ /**
+ * Used to get password from configuration, if default value is not available.
+ * @param conf configuration that contains password information
+ * @param key the key of the password
+ * @param val the default value of the key
+ * @return the value for the key
+ * @throws IOException if failed to get password from configuration
+ */
+ static public String getPassword(Configuration conf, String key, String val)
+ throws IOException {
+ if (StringUtils.isEmpty(val)) {
+ try {
+ final char[] pass = conf.getPassword(key);
+ if (pass != null) {
+ return (new String(pass)).trim();
+ } else {
+ return "";
+ }
+ } catch (IOException ioe) {
+ throw new IOException("Cannot find password option " + key, ioe);
+ }
+ } else {
+ return val;
+ }
+ }
+
+ /**
+ * Extract the user information details from a URI.
+ * @param name URI of the filesystem
+ * @return a login tuple, possibly empty.
+ */
+ public static UserInfo extractLoginDetails(URI name) {
+ try {
+ String authority = name.getAuthority();
+ if (authority == null) {
+ return UserInfo.EMPTY;
+ }
+ int loginIndex = authority.indexOf('@');
+ if (loginIndex < 0) {
+ // No user information
+ return UserInfo.EMPTY;
+ }
+ String login = authority.substring(0, loginIndex);
+ int loginSplit = login.indexOf(':');
+ if (loginSplit > 0) {
+ String user = login.substring(0, loginSplit);
+ String password = URLDecoder.decode(login.substring(loginSplit + 1),
+ "UTF-8");
+ return new UserInfo(user, password);
+ } else if (loginSplit == 0) {
+ // There is no user, just a password.
+ return UserInfo.EMPTY;
+ } else {
+ return new UserInfo(login, "");
+ }
+ } catch (UnsupportedEncodingException e) {
+ // This should never happen; translate it if it does.
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
new file mode 100644
index 0000000..4ee4cd4
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+/**
+ * ALL configuration constants for OSS filesystem.
+ */
+public final class Constants {
+
+ private Constants() {
+ }
+
+ // Class of credential provider
+ public static final String ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY =
+ "fs.oss.credentials.provider";
+
+ // OSS access verification
+ public static final String ACCESS_KEY = "fs.oss.access.key";
+ public static final String SECRET_KEY = "fs.oss.secret.key";
+
+ // Number of simultaneous connections to oss
+ public static final String MAXIMUM_CONNECTIONS_KEY =
+ "fs.oss.connection.maximum";
+ public static final int MAXIMUM_CONNECTIONS_DEFAULT = 32;
+
+ // Connect to oss over ssl
+ public static final String SECURE_CONNECTIONS_KEY =
+ "fs.oss.connection.secure.enabled";
+ public static final boolean SECURE_CONNECTIONS_DEFAULT = true;
+
+ // Use a custom endpoint
+ public static final String ENDPOINT_KEY = "fs.oss.endpoint";
+
+ // Connect to oss through a proxy server
+ public static final String PROXY_HOST_KEY = "fs.oss.proxy.host";
+ public static final String PROXY_PORT_KEY = "fs.oss.proxy.port";
+ public static final String PROXY_USERNAME_KEY = "fs.oss.proxy.username";
+ public static final String PROXY_PASSWORD_KEY = "fs.oss.proxy.password";
+ public static final String PROXY_DOMAIN_KEY = "fs.oss.proxy.domain";
+ public static final String PROXY_WORKSTATION_KEY =
+ "fs.oss.proxy.workstation";
+
+ // Number of times we should retry errors
+ public static final String MAX_ERROR_RETRIES_KEY = "fs.oss.attempts.maximum";
+ public static final int MAX_ERROR_RETRIES_DEFAULT = 20;
+
+ // Time until we give up trying to establish a connection to oss
+ public static final String ESTABLISH_TIMEOUT_KEY =
+ "fs.oss.connection.establish.timeout";
+ public static final int ESTABLISH_TIMEOUT_DEFAULT = 50000;
+
+ // Time until we give up on a connection to oss
+ public static final String SOCKET_TIMEOUT_KEY = "fs.oss.connection.timeout";
+ public static final int SOCKET_TIMEOUT_DEFAULT = 200000;
+
+ // Number of records to get while paging through a directory listing
+ public static final String MAX_PAGING_KEYS_KEY = "fs.oss.paging.maximum";
+ public static final int MAX_PAGING_KEYS_DEFAULT = 500;
+
+ // Size of each of or multipart pieces in bytes
+ public static final String MULTIPART_UPLOAD_SIZE_KEY =
+ "fs.oss.multipart.upload.size";
+
+ public static final long MULTIPART_UPLOAD_SIZE_DEFAULT = 10 * 1024 * 1024;
+ public static final int MULTIPART_UPLOAD_PART_NUM_LIMIT = 1000;
+
+ // Minimum size in bytes before we start a multipart uploads or copy
+ public static final String MIN_MULTIPART_UPLOAD_THRESHOLD_KEY =
+ "fs.oss.multipart.upload.threshold";
+ public static final long MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT =
+ 20 * 1024 * 1024;
+
+ public static final String MULTIPART_DOWNLOAD_SIZE_KEY =
+ "fs.oss.multipart.download.size";
+
+ public static final long MULTIPART_DOWNLOAD_SIZE_DEFAULT = 100 * 1024;
+
+ // Comma separated list of directories
+ public static final String BUFFER_DIR_KEY = "fs.oss.buffer.dir";
+
+ // private | public-read | public-read-write | authenticated-read |
+ // log-delivery-write | bucket-owner-read | bucket-owner-full-control
+ public static final String CANNED_ACL_KEY = "fs.oss.acl.default";
+ public static final String CANNED_ACL_DEFAULT = "";
+
+ // OSS server-side encryption
+ public static final String SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY =
+ "fs.oss.server-side-encryption-algorithm";
+
+ public static final String FS_OSS_BLOCK_SIZE_KEY = "fs.oss.block.size";
+ public static final int FS_OSS_BLOCK_SIZE_DEFAULT = 64 * 1024 * 1024;
+ public static final String FS_OSS = "oss";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java
new file mode 100644
index 0000000..234567b
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Aliyun OSS Filesystem.
+ */
+package org.apache.hadoop.fs.aliyun.oss;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java
new file mode 100644
index 0000000..37ed831
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.internal.AssumptionViolatedException;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Date;
+import java.util.Random;
+
+/**
+ * Utility class for OSS Tests.
+ */
+public final class OSSTestUtils {
+
+ private OSSTestUtils() {
+ }
+
+ /**
+ * Create the test filesystem.
+ *
+ * If the test.fs.oss.name property is not set,
+ * tests will fail.
+ *
+ * @param conf configuration
+ * @return the FS
+ * @throws IOException
+ */
+ public static AliyunOSSFileSystem createTestFileSystem(Configuration conf)
+ throws IOException {
+ String fsname = conf.getTrimmed(
+ TestOSSFileSystemContract.TEST_FS_OSS_NAME, "");
+
+ boolean liveTest = !StringUtils.isEmpty(fsname);
+ URI testURI = null;
+ if (liveTest) {
+ testURI = URI.create(fsname);
+ liveTest = testURI.getScheme().equals(Constants.FS_OSS);
+ }
+
+ if (!liveTest) {
+ throw new AssumptionViolatedException("No test filesystem in "
+ + TestOSSFileSystemContract.TEST_FS_OSS_NAME);
+ }
+ AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem();
+ ossfs.initialize(testURI, conf);
+ return ossfs;
+ }
+
+ /**
+ * Generate unique test path for multiple user tests.
+ *
+ * @return root test path
+ */
+ public static String generateUniqueTestPath() {
+ Long time = new Date().getTime();
+ Random rand = new Random();
+ return "/test_" + Long.toString(time) + "_"
+ + Long.toString(Math.abs(rand.nextLong()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java
new file mode 100644
index 0000000..de4e5a9
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Tests a live OSS system.
+ *
+ * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
+ * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
+ * properly making it impossible to skip the tests if we don't have a valid
+ * bucket.
+ */
+public class TestOSSFileSystemContract extends FileSystemContractBaseTest {
+
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestOSSFileSystemContract.class);
+
+ public static final String TEST_FS_OSS_NAME = "test.fs.oss.name";
+ private static String testRootPath = OSSTestUtils.generateUniqueTestPath();
+
+ @Override
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fs = OSSTestUtils.createTestFileSystem(conf);
+ super.setUp();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(super.path(testRootPath), true);
+ }
+ super.tearDown();
+ }
+
+ @Override
+ protected Path path(String path) {
+ if (path.startsWith("/")) {
+ return super.path(testRootPath + path);
+ } else {
+ return super.path(testRootPath + "/" + path);
+ }
+ }
+
+ @Override
+ public void testMkdirsWithUmask() throws Exception {
+ // not supported
+ }
+
+ /**
+ * Assert that root directory renames are not allowed.
+ *
+ * @throws Exception on failures
+ */
+ @Override
+ public void testRootDirAlwaysExists() throws Exception {
+ //this will throw an exception if the path is not found
+ fs.getFileStatus(super.path("/"));
+ //this catches overrides of the base exists() method that don't
+ //use getFileStatus() as an existence probe
+ assertTrue("FileSystem.exists() fails for root",
+ fs.exists(super.path("/")));
+ }
+
+ /**
+ * Assert that root directory renames are not allowed.
+ *
+ * @throws Exception on failures
+ */
+ @Override
+ public void testRenameRootDirForbidden() throws Exception {
+ if (!renameSupported()) {
+ return;
+ }
+ rename(super.path("/"),
+ super.path("/test/newRootDir"),
+ false, true, false);
+ }
+
+ public void testDeleteSubdir() throws IOException {
+ Path parentDir = this.path("/test/hadoop");
+ Path file = this.path("/test/hadoop/file");
+ Path subdir = this.path("/test/hadoop/subdir");
+ this.createFile(file);
+
+ assertTrue("Created subdir", this.fs.mkdirs(subdir));
+ assertTrue("File exists", this.fs.exists(file));
+ assertTrue("Parent dir exists", this.fs.exists(parentDir));
+ assertTrue("Subdir exists", this.fs.exists(subdir));
+
+ assertTrue("Deleted subdir", this.fs.delete(subdir, true));
+ assertTrue("Parent should exist", this.fs.exists(parentDir));
+
+ assertTrue("Deleted file", this.fs.delete(file, false));
+ assertTrue("Parent should exist", this.fs.exists(parentDir));
+ }
+
+
+ @Override
+ protected boolean renameSupported() {
+ return true;
+ }
+
+ @Override
+ public void testRenameNonExistentPath() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/path");
+ Path dst = this.path("/test/new/newpath");
+ try {
+ super.rename(src, dst, false, false, false);
+ fail("Should throw FileNotFoundException!");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+ }
+
+ @Override
+ public void testRenameFileMoveToNonExistentDirectory() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/file");
+ this.createFile(src);
+ Path dst = this.path("/test/new/newfile");
+ try {
+ super.rename(src, dst, false, true, false);
+ fail("Should throw FileNotFoundException!");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+ }
+
+ @Override
+ public void testRenameDirectoryMoveToNonExistentDirectory() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/dir");
+ this.fs.mkdirs(src);
+ Path dst = this.path("/test/new/newdir");
+ try {
+ super.rename(src, dst, false, true, false);
+ fail("Should throw FileNotFoundException!");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+ }
+
+ @Override
+ public void testRenameFileMoveToExistingDirectory() throws Exception {
+ super.testRenameFileMoveToExistingDirectory();
+ }
+
+ @Override
+ public void testRenameFileAsExistingFile() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/file");
+ this.createFile(src);
+ Path dst = this.path("/test/new/newfile");
+ this.createFile(dst);
+ try {
+ super.rename(src, dst, false, true, true);
+ fail("Should throw FileAlreadyExistsException");
+ } catch (FileAlreadyExistsException e) {
+ // expected
+ }
+ }
+ }
+
+ @Override
+ public void testRenameDirectoryAsExistingFile() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/dir");
+ this.fs.mkdirs(src);
+ Path dst = this.path("/test/new/newfile");
+ this.createFile(dst);
+ try {
+ super.rename(src, dst, false, true, true);
+ fail("Should throw FileAlreadyExistsException");
+ } catch (FileAlreadyExistsException e) {
+ // expected
+ }
+ }
+ }
+
+ public void testGetFileStatusFileAndDirectory() throws Exception {
+ Path filePath = this.path("/test/oss/file1");
+ this.createFile(filePath);
+ assertTrue("Should be file", this.fs.getFileStatus(filePath).isFile());
+ assertFalse("Should not be directory",
+ this.fs.getFileStatus(filePath).isDirectory());
+
+ Path dirPath = this.path("/test/oss/dir");
+ this.fs.mkdirs(dirPath);
+ assertTrue("Should be directory",
+ this.fs.getFileStatus(dirPath).isDirectory());
+ assertFalse("Should not be file", this.fs.getFileStatus(dirPath).isFile());
+ }
+
+ public void testMkdirsForExistingFile() throws Exception {
+ Path testFile = this.path("/test/hadoop/file");
+ assertFalse(this.fs.exists(testFile));
+ this.createFile(testFile);
+ assertTrue(this.fs.exists(testFile));
+ try {
+ this.fs.mkdirs(testFile);
+ fail("Should throw FileAlreadyExistsException!");
+ } catch (FileAlreadyExistsException e) {
+ // expected
+ }
+ }
+
+ public void testWorkingDirectory() throws Exception {
+ Path workDir = super.path(this.getDefaultWorkingDirectory());
+ assertEquals(workDir, this.fs.getWorkingDirectory());
+ this.fs.setWorkingDirectory(super.path("."));
+ assertEquals(workDir, this.fs.getWorkingDirectory());
+ this.fs.setWorkingDirectory(super.path(".."));
+ assertEquals(workDir.getParent(), this.fs.getWorkingDirectory());
+ Path relativeDir = super.path("hadoop");
+ this.fs.setWorkingDirectory(relativeDir);
+ assertEquals(relativeDir, this.fs.getWorkingDirectory());
+ Path absoluteDir = super.path("/test/hadoop");
+ this.fs.setWorkingDirectory(absoluteDir);
+ assertEquals(absoluteDir, this.fs.getWorkingDirectory());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5d53422/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java
new file mode 100644
index 0000000..411cd57
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.*;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Random;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests basic functionality for AliyunOSSInputStream, including seeking and
+ * reading files.
+ */
+public class TestOSSInputStream {
+
+ private FileSystem fs;
+
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestOSSInputStream.class);
+
+ private static String testRootPath = OSSTestUtils.generateUniqueTestPath();
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fs = OSSTestUtils.createTestFileSystem(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(new Path(testRootPath), true);
+ }
+ }
+
+ private Path setPath(String path) {
+ if (path.startsWith("/")) {
+ return new Path(testRootPath + path);
+ } else {
+ return new Path(testRootPath + "/" + path);
+ }
+ }
+
+ @Test
+ public void testSeekFile() throws Exception {
+ Path smallSeekFile = setPath("/test/smallSeekFile.txt");
+ long size = 5 * 1024 * 1024;
+
+ ContractTestUtils.generateTestFile(this.fs, smallSeekFile, size, 256, 255);
+ LOG.info("5MB file created: smallSeekFile.txt");
+
+ FSDataInputStream instream = this.fs.open(smallSeekFile);
+ int seekTimes = 5;
+ LOG.info("multiple fold position seeking test...:");
+ for (int i = 0; i < seekTimes; i++) {
+ long pos = size / (seekTimes - i) - 1;
+ LOG.info("begin seeking for pos: " + pos);
+ instream.seek(pos);
+ assertTrue("expected position at:" + pos + ", but got:"
+ + instream.getPos(), instream.getPos() == pos);
+ LOG.info("completed seeking at pos: " + instream.getPos());
+ }
+ LOG.info("random position seeking test...:");
+ Random rand = new Random();
+ for (int i = 0; i < seekTimes; i++) {
+ long pos = Math.abs(rand.nextLong()) % size;
+ LOG.info("begin seeking for pos: " + pos);
+ instream.seek(pos);
+ assertTrue("expected position at:" + pos + ", but got:"
+ + instream.getPos(), instream.getPos() == pos);
+ LOG.info("completed seeking at pos: " + instream.getPos());
+ }
+ IOUtils.closeStream(instream);
+ }
+
+ @Test
+ public void testReadFile() throws Exception {
+ final int bufLen = 256;
+ final int sizeFlag = 5;
+ String filename = "readTestFile_" + sizeFlag + ".txt";
+ Path readTestFile = setPath("/test/" + filename);
+ long size = sizeFlag * 1024 * 1024;
+
+ ContractTestUtils.generateTestFile(this.fs, readTestFile, size, 256, 255);
+ LOG.info(sizeFlag + "MB file created: /test/" + filename);
+
+ FSDataInputStream instream = this.fs.open(readTestFile);
+ byte[] buf = new byte[bufLen];
+ long bytesRead = 0;
+ while (bytesRead < size) {
+ int bytes;
+ if (size - bytesRead < bufLen) {
+ int remaining = (int)(size - bytesRead);
+ bytes = instream.read(buf, 0, remaining);
+ } else {
+ bytes = instream.read(buf, 0, bufLen);
+ }
+ bytesRead += bytes;
+
+ if (bytesRead % (1024 * 1024) == 0) {
+ int available = instream.available();
+ int remaining = (int)(size - bytesRead);
+ assertTrue("expected remaining:" + remaining + ", but got:" + available,
+ remaining == available);
+ LOG.info("Bytes read: " + Math.round((double)bytesRead / (1024 * 1024))
+ + " MB");
+ }
+ }
+ assertTrue(instream.available() == 0);
+ IOUtils.closeStream(instream);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/22] hadoop git commit: HDFS-9847. HDFS configuration should
accept time units. Contributed by Yiqun Lin
Posted by sh...@apache.org.
HDFS-9847. HDFS configuration should accept time units. Contributed by Yiqun Lin
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d37dc5d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d37dc5d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d37dc5d1
Branch: refs/heads/HADOOP-12756
Commit: d37dc5d1b8e022a7085118a2e7066623483c293f
Parents: f388a20
Author: Chris Douglas <cd...@apache.org>
Authored: Tue Sep 6 10:37:54 2016 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Tue Sep 6 10:38:04 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 47 ++++++++++++++------
.../apache/hadoop/conf/TestConfiguration.java | 46 +++++++++++++++++++
.../hadoop/hdfs/client/impl/DfsClientConf.java | 6 ++-
.../hadoop/hdfs/server/balancer/Balancer.java | 12 +++--
.../server/blockmanagement/BlockManager.java | 6 ++-
.../BlockPlacementPolicyDefault.java | 5 ++-
.../server/blockmanagement/DatanodeManager.java | 9 ++--
.../blockmanagement/DecommissionManager.java | 7 +--
.../hadoop/hdfs/server/datanode/DNConf.java | 18 ++++----
.../hadoop/hdfs/server/datanode/DataNode.java | 4 +-
.../hdfs/server/datanode/DirectoryScanner.java | 6 ++-
.../apache/hadoop/hdfs/server/mover/Mover.java | 12 +++--
.../hdfs/server/namenode/CheckpointConf.java | 10 +++--
.../hadoop/hdfs/server/namenode/FSImage.java | 7 +--
.../hdfs/server/namenode/ha/EditLogTailer.java | 10 +++--
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 6 ++-
.../src/main/resources/hdfs-default.xml | 42 ++++++++++++++---
17 files changed, 187 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 29733eb..66734b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -630,10 +630,15 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
ArrayList<String > names = new ArrayList<String>();
if (isDeprecated(name)) {
DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
- warnOnceIfDeprecated(deprecations, name);
- for (String newKey : keyInfo.newKeys) {
- if(newKey != null) {
- names.add(newKey);
+ if (keyInfo != null) {
+ if (!keyInfo.getAndSetAccessed()) {
+ logDeprecation(keyInfo.getWarningMessage(name));
+ }
+
+ for (String newKey : keyInfo.newKeys) {
+ if (newKey != null) {
+ names.add(newKey);
+ }
}
}
}
@@ -1232,11 +1237,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
}
- private void warnOnceIfDeprecated(DeprecationContext deprecations, String name) {
- DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
- if (keyInfo != null && !keyInfo.getAndSetAccessed()) {
- LOG_DEPRECATION.info(keyInfo.getWarningMessage(name));
- }
+ @VisibleForTesting
+ void logDeprecation(String message) {
+ LOG_DEPRECATION.info(message);
}
/**
@@ -1625,20 +1628,38 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
String vStr = get(name);
if (null == vStr) {
return defaultValue;
+ } else {
+ return getTimeDurationHelper(name, vStr, unit);
+ }
+ }
+
+ public long getTimeDuration(String name, String defaultValue, TimeUnit unit) {
+ String vStr = get(name);
+ if (null == vStr) {
+ return getTimeDurationHelper(name, defaultValue, unit);
+ } else {
+ return getTimeDurationHelper(name, vStr, unit);
}
- vStr = vStr.trim();
- return getTimeDurationHelper(name, vStr, unit);
}
private long getTimeDurationHelper(String name, String vStr, TimeUnit unit) {
+ vStr = vStr.trim();
+ vStr = StringUtils.toLowerCase(vStr);
ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr);
if (null == vUnit) {
- LOG.warn("No unit for " + name + "(" + vStr + ") assuming " + unit);
+ logDeprecation("No unit for " + name + "(" + vStr + ") assuming " + unit);
vUnit = ParsedTimeDuration.unitFor(unit);
} else {
vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix()));
}
- return unit.convert(Long.parseLong(vStr), vUnit.unit());
+
+ long raw = Long.parseLong(vStr);
+ long converted = unit.convert(raw, vUnit.unit());
+ if (vUnit.unit().convert(converted, unit) < raw) {
+ logDeprecation("Possible loss of precision converting " + vStr
+ + vUnit.suffix() + " to " + unit + " for " + name);
+ }
+ return converted;
}
public long[] getTimeDurations(String name, TimeUnit unit) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index aa3b239..e7ebea1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -880,6 +880,15 @@ public class TestConfiguration extends TestCase {
assertEquals(30L, conf.getTimeDuration("test.time.X", 30, SECONDS));
conf.set("test.time.X", "30");
assertEquals(30L, conf.getTimeDuration("test.time.X", 40, SECONDS));
+ assertEquals(10L, conf.getTimeDuration("test.time.c", "10", SECONDS));
+ assertEquals(30L, conf.getTimeDuration("test.time.c", "30s", SECONDS));
+ assertEquals(120L, conf.getTimeDuration("test.time.c", "2m", SECONDS));
+ conf.set("test.time.c", "30");
+ assertEquals(30L, conf.getTimeDuration("test.time.c", "40s", SECONDS));
+
+ // check suffix insensitive
+ conf.set("test.time.d", "30S");
+ assertEquals(30L, conf.getTimeDuration("test.time.d", 40, SECONDS));
for (Configuration.ParsedTimeDuration ptd :
Configuration.ParsedTimeDuration.values()) {
@@ -889,6 +898,43 @@ public class TestConfiguration extends TestCase {
}
}
+ public void testTimeDurationWarning() {
+ // check warn for possible loss of precision
+ final String warnFormat = "Possible loss of precision converting %s" +
+ " to %s for test.time.warn";
+ final ArrayList<String> warnchk = new ArrayList<>();
+ Configuration wconf = new Configuration(false) {
+ @Override
+ void logDeprecation(String message) {
+ warnchk.add(message);
+ }
+ };
+ String[] convDAYS = new String[]{"23h", "30m", "40s", "10us", "40000ms"};
+ for (String s : convDAYS) {
+ wconf.set("test.time.warn", s);
+ assertEquals(0, wconf.getTimeDuration("test.time.warn", 1, DAYS));
+ }
+ for (int i = 0; i < convDAYS.length; ++i) {
+ String wchk = String.format(warnFormat, convDAYS[i], "DAYS");
+ assertEquals(wchk, warnchk.get(i));
+ }
+
+ warnchk.clear();
+ wconf.setTimeDuration("test.time.warn", 1441, MINUTES);
+ assertEquals(1, wconf.getTimeDuration("test.time.warn", 0, DAYS));
+ assertEquals(24, wconf.getTimeDuration("test.time.warn", 0, HOURS));
+ String dchk = String.format(warnFormat, "1441m", "DAYS");
+ assertEquals(dchk, warnchk.get(0));
+ String hchk = String.format(warnFormat, "1441m", "HOURS");
+ assertEquals(hchk, warnchk.get(1));
+ assertEquals(1441, wconf.getTimeDuration("test.time.warn", 0, MINUTES));
+ // no warning
+ assertEquals(2, warnchk.size());
+ assertEquals(86460, wconf.getTimeDuration("test.time.warn", 0, SECONDS));
+ // no warning
+ assertEquals(2, warnchk.size());
+ }
+
public void testPattern() throws IOException {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index ded989c..4f4c7b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -85,6 +85,7 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
/**
* DFSClient configuration.
@@ -233,9 +234,10 @@ public class DfsClientConf {
connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME,
DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
- datanodeRestartTimeout = conf.getLong(
+ datanodeRestartTimeout = conf.getTimeDuration(
DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY,
- DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT) * 1000;
+ DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT,
+ TimeUnit.SECONDS) * 1000;
slowIoWarningThresholdMs = conf.getLong(
DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY,
DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 8d94ed5..2037d01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -33,6 +33,7 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -662,10 +663,13 @@ public class Balancer {
static int run(Collection<URI> namenodes, final BalancerParameters p,
Configuration conf) throws IOException, InterruptedException {
final long sleeptime =
- conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 2000 +
- conf.getLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT) * 1000;
+ conf.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 2000 +
+ conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 1000;
LOG.info("namenodes = " + namenodes);
LOG.info("parameters = " + p);
LOG.info("included nodes = " + p.getIncludedNodes());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 9e3b935..1362c0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -42,6 +42,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import javax.management.ObjectName;
@@ -405,8 +406,9 @@ public class BlockManager implements BlockStatsMXBean {
this.blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
this.replicationRecheckInterval =
- conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT) * 1000L;
+ conf.getTimeDuration(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 1000L;
this.storageInfoDefragmentInterval =
conf.getLong(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 19f4dde..abfa782 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import static org.apache.hadoop.util.Time.monotonicNow;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -89,9 +90,9 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
this.stats = stats;
this.clusterMap = clusterMap;
this.host2datanodeMap = host2datanodeMap;
- this.heartbeatInterval = conf.getLong(
+ this.heartbeatInterval = conf.getTimeDuration(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 1000;
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 1000;
this.tolerateHeartbeatMultiplier = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_KEY,
DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index fffe29c..2d6547f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -58,6 +58,7 @@ import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
/**
* Manage datanodes, include decommission and other activities.
@@ -234,9 +235,9 @@ public class DatanodeManager {
dnsToSwitchMapping.resolve(locations);
}
- heartbeatIntervalSeconds = conf.getLong(
+ heartbeatIntervalSeconds = conf.getTimeDuration(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
heartbeatRecheckInterval = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 minutes
@@ -289,9 +290,9 @@ public class DatanodeManager {
" = '" + staleInterval + "' is invalid. " +
"It should be a positive non-zero value.");
- final long heartbeatIntervalSeconds = conf.getLong(
+ final long heartbeatIntervalSeconds = conf.getTimeDuration(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
// The stale interval value cannot be smaller than
// 3 times of heartbeat interval
final long minStaleInterval = conf.getInt(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index c456aba..6436fab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -134,9 +134,10 @@ public class DecommissionManager {
* @param conf
*/
void activate(Configuration conf) {
- final int intervalSecs =
- conf.getInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_DEFAULT);
+ final int intervalSecs = (int) conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS);
checkArgument(intervalSecs >= 0, "Cannot set a negative " +
"value for " + DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 11197e6..757d7f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -64,6 +64,8 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil;
import org.apache.hadoop.security.SaslPropertiesResolver;
+import java.util.concurrent.TimeUnit;
+
/**
* Simple class encapsulating all of the configuration that the DataNode
* loads at startup time.
@@ -184,9 +186,9 @@ public class DNConf {
DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY,
DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
- long initBRDelay = conf.getLong(
+ long initBRDelay = conf.getTimeDuration(
DFS_BLOCKREPORT_INITIAL_DELAY_KEY,
- DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT) * 1000L;
+ DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT, TimeUnit.SECONDS) * 1000L;
if (initBRDelay >= blockReportInterval) {
initBRDelay = 0;
DataNode.LOG.info("dfs.blockreport.initialDelay is "
@@ -195,12 +197,12 @@ public class DNConf {
}
initialBlockReportDelayMs = initBRDelay;
- heartBeatInterval = conf.getLong(DFS_HEARTBEAT_INTERVAL_KEY,
- DFS_HEARTBEAT_INTERVAL_DEFAULT) * 1000L;
+ heartBeatInterval = conf.getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY,
+ DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 1000L;
long confLifelineIntervalMs =
conf.getLong(DFS_DATANODE_LIFELINE_INTERVAL_SECONDS_KEY,
- 3 * conf.getLong(DFS_HEARTBEAT_INTERVAL_KEY,
- DFS_HEARTBEAT_INTERVAL_DEFAULT)) * 1000L;
+ 3 * conf.getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY,
+ DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS)) * 1000L;
if (confLifelineIntervalMs <= heartBeatInterval) {
confLifelineIntervalMs = 3 * heartBeatInterval;
DataNode.LOG.warn(
@@ -245,9 +247,9 @@ public class DNConf {
DFS_DATANODE_NON_LOCAL_LAZY_PERSIST,
DFS_DATANODE_NON_LOCAL_LAZY_PERSIST_DEFAULT);
- this.bpReadyTimeout = conf.getLong(
+ this.bpReadyTimeout = conf.getTimeDuration(
DFS_DATANODE_BP_READY_TIMEOUT_KEY,
- DFS_DATANODE_BP_READY_TIMEOUT_DEFAULT);
+ DFS_DATANODE_BP_READY_TIMEOUT_DEFAULT, TimeUnit.SECONDS);
this.volFailuresTolerated =
conf.getInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 08a1fc0..0025040 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1024,8 +1024,8 @@ public class DataNode extends ReconfigurableBase
return;
}
String reason = null;
- if (conf.getInt(DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
- DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT) < 0) {
+ if (conf.getTimeDuration(DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
+ DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT, TimeUnit.SECONDS) < 0) {
reason = "verification is turned off by configuration";
} else if ("SimulatedFSDataset".equals(data.getClass().getSimpleName())) {
reason = "verifcation is not supported by SimulatedFSDataset";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 7c8985e..f9ebab9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -414,8 +414,10 @@ public class DirectoryScanner implements Runnable {
DirectoryScanner(DataNode datanode, FsDatasetSpi<?> dataset, Configuration conf) {
this.datanode = datanode;
this.dataset = dataset;
- int interval = conf.getInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
- DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT);
+ int interval = (int) conf.getTimeDuration(
+ DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
+ DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS);
scanPeriodMsecs = interval * MILLIS_PER_SECOND; //msec
int throttle =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index fdb6cfa..0c85374 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -64,6 +64,7 @@ import java.net.InetSocketAddress;
import java.net.URI;
import java.text.DateFormat;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@InterfaceAudience.Private
@@ -602,10 +603,13 @@ public class Mover {
static int run(Map<URI, List<Path>> namenodes, Configuration conf)
throws IOException, InterruptedException {
final long sleeptime =
- conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 2000 +
- conf.getLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT) * 1000;
+ conf.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 2000 +
+ conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 1000;
AtomicInteger retryCount = new AtomicInteger(0);
LOG.info("namenodes = " + namenodes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
index c30730b..aeee87d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.conf.Configuration;
import com.google.common.collect.ImmutableList;
+import java.util.concurrent.TimeUnit;
+
@InterfaceAudience.Private
public class CheckpointConf {
private static final Log LOG = LogFactory.getLog(CheckpointConf.class);
@@ -52,12 +54,12 @@ public class CheckpointConf {
private double quietMultiplier;
public CheckpointConf(Configuration conf) {
- checkpointCheckPeriod = conf.getLong(
+ checkpointCheckPeriod = conf.getTimeDuration(
DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY,
- DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_DEFAULT);
+ DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_DEFAULT, TimeUnit.SECONDS);
- checkpointPeriod = conf.getLong(DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
- DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT);
+ checkpointPeriod = conf.getTimeDuration(DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
+ DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT, TimeUnit.SECONDS);
checkpointTxnCount = conf.getLong(DFS_NAMENODE_CHECKPOINT_TXNS_KEY,
DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT);
maxRetriesOnMergeError = conf.getInt(DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index b637105..d02a6ca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -34,6 +34,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -798,9 +799,9 @@ public class FSImage implements Closeable {
*/
private boolean needsResaveBasedOnStaleCheckpoint(
File imageFile, long numEditsLoaded) {
- final long checkpointPeriod = conf.getLong(
- DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
- DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT);
+ final long checkpointPeriod = conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
+ DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT, TimeUnit.SECONDS);
final long checkpointTxnCount = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY,
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
index 95c3d58..f57cb4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
@@ -146,8 +146,9 @@ public class EditLogTailer {
lastLoadTimeMs = monotonicNow();
- logRollPeriodMs = conf.getInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY,
- DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT) * 1000;
+ logRollPeriodMs = conf.getTimeDuration(
+ DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY,
+ DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT, TimeUnit.SECONDS) * 1000;
List<RemoteNameNodeInfo> nns = Collections.emptyList();
if (logRollPeriodMs >= 0) {
try {
@@ -172,8 +173,9 @@ public class EditLogTailer {
DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY + " is negative.");
}
- sleepTimeMs = conf.getInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,
- DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT) * 1000;
+ sleepTimeMs = conf.getTimeDuration(
+ DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,
+ DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT, TimeUnit.SECONDS) * 1000;
rollEditsTimeoutMs = conf.getInt(
DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index ba09e8b..e6e0fbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -34,6 +34,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
@@ -742,9 +743,10 @@ public class DFSAdmin extends FsShell {
long timeWindow = 0;
long txGap = 0;
if (argv.length > 1 && "-beforeShutdown".equals(argv[1])) {
- final long checkpointPeriod = dfsConf.getLong(
+ final long checkpointPeriod = dfsConf.getTimeDuration(
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
- DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT);
+ DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT,
+ TimeUnit.SECONDS);
final long checkpointTxnCount = dfsConf.getLong(
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY,
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 0c2c743..4eca80c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -637,8 +637,12 @@
</property>
<property>
- <name>dfs.blockreport.initialDelay</name> <value>0</value>
- <description>Delay for first block report in seconds.</description>
+ <name>dfs.blockreport.initialDelay</name>
+ <value>0</value>
+ <description>
+ Delay for first block report in seconds. Support multiple time unit
+ suffix(case insensitive), as described in dfs.heartbeat.interval.
+ </description>
</property>
<property>
@@ -681,6 +685,8 @@
<value>21600</value>
<description>Interval in seconds for Datanode to scan data directories and
reconcile the difference between blocks in memory and on the disk.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -715,7 +721,13 @@
<property>
<name>dfs.heartbeat.interval</name>
<value>3</value>
- <description>Determines datanode heartbeat interval in seconds.</description>
+ <description>
+ Determines datanode heartbeat interval in seconds.
+ Can use the following suffix (case insensitive):
+ ms(millis), s(sec), m(min), h(hour), d(day)
+ to specify the time (such as 2s, 2m, 1h, etc.).
+ Or provide complete number in seconds (such as 30 for 30 seconds).
+ </description>
</property>
<property>
@@ -920,7 +932,9 @@
<name>dfs.namenode.decommission.interval</name>
<value>30</value>
<description>Namenode periodicity in seconds to check if decommission is
- complete.</description>
+ complete. Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
+ </description>
</property>
<property>
@@ -949,7 +963,9 @@
<name>dfs.namenode.replication.interval</name>
<value>3</value>
<description>The periodicity in seconds with which the namenode computes
- replication work for datanodes. </description>
+ replication work for datanodes. Support multiple time unit suffix(case insensitive),
+ as described in dfs.heartbeat.interval.
+ </description>
</property>
<property>
@@ -1044,7 +1060,10 @@
<property>
<name>dfs.namenode.checkpoint.period</name>
<value>3600</value>
- <description>The number of seconds between two periodic checkpoints.
+ <description>
+ The number of seconds between two periodic checkpoints.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -1062,7 +1081,8 @@
<value>60</value>
<description>The SecondaryNameNode and CheckpointNode will poll the NameNode
every 'dfs.namenode.checkpoint.check.period' seconds to query the number
- of uncheckpointed transactions.
+ of uncheckpointed transactions. Support multiple time unit suffix(case insensitive),
+ as described in dfs.heartbeat.interval.
</description>
</property>
@@ -1408,6 +1428,8 @@
the datanode dead and invoking the normal recovery mechanisms.
The notification is sent by a datanode when it is being shutdown
using the shutdownDatanode admin command with the upgrade option.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -1475,6 +1497,8 @@
log segments, the StandbyNode will only be as up-to-date as how
often the logs are rolled. Note that failover triggers a log roll
so the StandbyNode will be up to date before it becomes active.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -1484,6 +1508,8 @@
<description>
How often, in seconds, the StandbyNode should check for new
finalized log segments in the shared edits log.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -2918,6 +2944,8 @@
received request. Setting this to 0 fails requests right away if the
datanode is not yet registered with the namenode. This wait time
reduces initial request failures after datanode restart.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/22] hadoop git commit: Add CHANGES and release notes for
3.0.0-alpha1 to site
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f388a209/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/CHANGES.3.0.0-alpha1.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/CHANGES.3.0.0-alpha1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/CHANGES.3.0.0-alpha1.md
new file mode 100644
index 0000000..e539a80
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/CHANGES.3.0.0-alpha1.md
@@ -0,0 +1,3712 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# "Apache Hadoop" Changelog
+
+## Release 3.0.0-alpha1 - 2016-08-30
+
+### INCOMPATIBLE CHANGES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-8124](https://issues.apache.org/jira/browse/HADOOP-8124) | Remove the deprecated Syncable.sync() method | Major | fs | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HDFS-3034](https://issues.apache.org/jira/browse/HDFS-3034) | Remove the deprecated Syncable.sync() method | Major | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HDFS-5079](https://issues.apache.org/jira/browse/HDFS-5079) | Cleaning up NNHAStatusHeartbeat.State DatanodeProtocolProtos. | Major | datanode, namenode | Konstantin Shvachko | Tao Luo |
+| [HDFS-5570](https://issues.apache.org/jira/browse/HDFS-5570) | Deprecate hftp / hsftp and replace them with webhdfs / swebhdfs | Major | . | Haohui Mai | Haohui Mai |
+| [HADOOP-10485](https://issues.apache.org/jira/browse/HADOOP-10485) | Remove dead classes in hadoop-streaming | Major | tools | Haohui Mai | Haohui Mai |
+| [HDFS-6246](https://issues.apache.org/jira/browse/HDFS-6246) | Remove 'dfs.support.append' flag from trunk code | Minor | . | Uma Maheswara Rao G | Uma Maheswara Rao G |
+| [HADOOP-10474](https://issues.apache.org/jira/browse/HADOOP-10474) | Move o.a.h.record to hadoop-streaming | Major | . | Haohui Mai | Haohui Mai |
+| [HDFS-2538](https://issues.apache.org/jira/browse/HDFS-2538) | option to disable fsck dots | Minor | . | Allen Wittenauer | Mohammad Kamrul Islam |
+| [HADOOP-9902](https://issues.apache.org/jira/browse/HADOOP-9902) | Shell script rewrite | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11041](https://issues.apache.org/jira/browse/HADOOP-11041) | VersionInfo output specifies subversion | Minor | build | Guo Ruijing | Tsuyoshi Ozawa |
+| [MAPREDUCE-5972](https://issues.apache.org/jira/browse/MAPREDUCE-5972) | Fix typo 'programatically' in job.xml (and a few other places) | Trivial | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-11356](https://issues.apache.org/jira/browse/HADOOP-11356) | Removed deprecated o.a.h.fs.permission.AccessControlException | Major | . | Haohui Mai | Li Lu |
+| [HADOOP-10950](https://issues.apache.org/jira/browse/HADOOP-10950) | rework heap management vars | Major | scripts | Allen Wittenauer | John Smith |
+| [MAPREDUCE-5785](https://issues.apache.org/jira/browse/MAPREDUCE-5785) | Derive heap size or mapreduce.\*.memory.mb automatically | Major | mr-am, task | Gera Shegalov | Gera Shegalov |
+| [YARN-2428](https://issues.apache.org/jira/browse/YARN-2428) | LCE default banned user list should have yarn | Trivial | nodemanager | Allen Wittenauer | Varun Saxena |
+| [MAPREDUCE-5653](https://issues.apache.org/jira/browse/MAPREDUCE-5653) | DistCp does not honour config-overrides for mapreduce.[map,reduce].memory.mb | Major | distcp | Mithun Radhakrishnan | Ratandeep Ratti |
+| [HADOOP-11657](https://issues.apache.org/jira/browse/HADOOP-11657) | Align the output of `hadoop fs -du` to be more Unix-like | Minor | . | Nick Dimiduk | Akira Ajisaka |
+| [HDFS-7302](https://issues.apache.org/jira/browse/HDFS-7302) | namenode -rollingUpgrade downgrade may finalize a rolling upgrade | Major | namenode | Tsz Wo Nicholas Sze | Kai Sasaki |
+| [HADOOP-6857](https://issues.apache.org/jira/browse/HADOOP-6857) | FsShell should report raw disk usage including replication factor | Major | fs | Alex Kozlov | Byron Wong |
+| [HADOOP-10115](https://issues.apache.org/jira/browse/HADOOP-10115) | Exclude duplicate jars in hadoop package under different component's lib | Major | build | Vinayakumar B | Vinayakumar B |
+| [YARN-3154](https://issues.apache.org/jira/browse/YARN-3154) | Should not upload partial logs for MR jobs or other "short-running' applications | Blocker | nodemanager, resourcemanager | Xuan Gong | Xuan Gong |
+| [MAPREDUCE-4424](https://issues.apache.org/jira/browse/MAPREDUCE-4424) | 'mapred job -list' command should show the job name as well | Minor | mrv2 | Harsh J | Avinash Kujur |
+| [YARN-3241](https://issues.apache.org/jira/browse/YARN-3241) | FairScheduler handles "invalid" queue names inconsistently | Major | fairscheduler | zhihai xu | zhihai xu |
+| [HDFS-7985](https://issues.apache.org/jira/browse/HDFS-7985) | WebHDFS should be always enabled | Major | . | Haohui Mai | Li Lu |
+| [HDFS-6353](https://issues.apache.org/jira/browse/HDFS-6353) | Check and make checkpoint before stopping the NameNode | Major | namenode | Suresh Srinivas | Jing Zhao |
+| [HADOOP-11553](https://issues.apache.org/jira/browse/HADOOP-11553) | Formalize the shell API | Blocker | documentation, scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11731](https://issues.apache.org/jira/browse/HADOOP-11731) | Rework the changelog and releasenotes | Major | documentation | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11746](https://issues.apache.org/jira/browse/HADOOP-11746) | rewrite test-patch.sh | Major | build, test | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11627](https://issues.apache.org/jira/browse/HADOOP-11627) | Remove io.native.lib.available | Major | native | Akira Ajisaka | Brahma Reddy Battula |
+| [HDFS-7281](https://issues.apache.org/jira/browse/HDFS-7281) | Missing block is marked as corrupted block | Major | namenode | Ming Ma | Ming Ma |
+| [HDFS-8226](https://issues.apache.org/jira/browse/HDFS-8226) | Non-HA rollback compatibility broken | Blocker | . | J.Andreina | J.Andreina |
+| [MAPREDUCE-2632](https://issues.apache.org/jira/browse/MAPREDUCE-2632) | Avoid calling the partitioner when the numReduceTasks is 1. | Major | . | Ravi Teja Ch N V | Sunil G |
+| [HDFS-8241](https://issues.apache.org/jira/browse/HDFS-8241) | Remove unused NameNode startup option -finalize | Minor | namenode | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-8332](https://issues.apache.org/jira/browse/HDFS-8332) | DFS client API calls should check filesystem closed | Major | . | Rakesh R | Rakesh R |
+| [HADOOP-11698](https://issues.apache.org/jira/browse/HADOOP-11698) | Remove DistCpV1 and Logalyzer | Major | tools/distcp | Allen Wittenauer | Brahma Reddy Battula |
+| [HADOOP-11772](https://issues.apache.org/jira/browse/HADOOP-11772) | RPC Invoker relies on static ClientCache which has synchronized(this) blocks | Major | ipc, performance | Gopal V | Haohui Mai |
+| [YARN-2336](https://issues.apache.org/jira/browse/YARN-2336) | Fair scheduler REST api returns a missing '[' bracket JSON for deep queue tree | Major | fairscheduler | Kenji Kikushima | Akira Ajisaka |
+| [HDFS-8135](https://issues.apache.org/jira/browse/HDFS-8135) | Remove the deprecated FSConstants class | Major | . | Haohui Mai | Li Lu |
+| [MAPREDUCE-6336](https://issues.apache.org/jira/browse/MAPREDUCE-6336) | Enable v2 FileOutputCommitter by default | Major | mrv2 | Gera Shegalov | Siqi Li |
+| [YARN-41](https://issues.apache.org/jira/browse/YARN-41) | The RM should handle the graceful shutdown of the NM. | Major | nodemanager, resourcemanager | Ravi Teja Ch N V | Devaraj K |
+| [HDFS-8591](https://issues.apache.org/jira/browse/HDFS-8591) | Remove support for deprecated configuration key dfs.namenode.decommission.nodes.per.interval | Minor | namenode | Andrew Wang | Andrew Wang |
+| [HDFS-6564](https://issues.apache.org/jira/browse/HDFS-6564) | Use slf4j instead of common-logging in hdfs-client | Major | build | Haohui Mai | Rakesh R |
+| [MAPREDUCE-6427](https://issues.apache.org/jira/browse/MAPREDUCE-6427) | Fix typo in JobHistoryEventHandler | Minor | . | Brahma Reddy Battula | Ray Chiang |
+| [HADOOP-12269](https://issues.apache.org/jira/browse/HADOOP-12269) | Update aws-sdk dependency to 1.10.6; move to aws-sdk-s3 | Major | fs/s3 | Thomas Demoor | Thomas Demoor |
+| [HDFS-8895](https://issues.apache.org/jira/browse/HDFS-8895) | Remove deprecated BlockStorageLocation APIs | Major | . | Andrew Wang | Andrew Wang |
+| [HDFS-8900](https://issues.apache.org/jira/browse/HDFS-8900) | Compact XAttrs to optimize memory footprint. | Major | namenode | Yi Liu | Yi Liu |
+| [HDFS-8981](https://issues.apache.org/jira/browse/HDFS-8981) | Adding revision to data node jmx getVersion() method | Minor | . | Siqi Li | Siqi Li |
+| [YARN-4087](https://issues.apache.org/jira/browse/YARN-4087) | Followup fixes after YARN-2019 regarding RM behavior when state-store error occurs | Major | . | Jian He | Jian He |
+| [HADOOP-12416](https://issues.apache.org/jira/browse/HADOOP-12416) | Trash messages should be handled by Logger instead of being delivered on System.out | Major | trash | Ashutosh Chauhan | Mingliang Liu |
+| [HDFS-9085](https://issues.apache.org/jira/browse/HDFS-9085) | Show renewer information in DelegationTokenIdentifier#toString | Trivial | security | zhihai xu | zhihai xu |
+| [HADOOP-12495](https://issues.apache.org/jira/browse/HADOOP-12495) | Fix posix\_spawn error on OS X | Major | util | Allen Wittenauer | Allen Wittenauer |
+| [HDFS-9070](https://issues.apache.org/jira/browse/HDFS-9070) | Allow fsck display pending replica location information for being-written blocks | Major | . | Rui Gao | Rui Gao |
+| [HDFS-9278](https://issues.apache.org/jira/browse/HDFS-9278) | Fix preferredBlockSize typo in OIV XML output | Trivial | tools | Nicole Pazmany | Nicole Pazmany |
+| [HADOOP-10787](https://issues.apache.org/jira/browse/HADOOP-10787) | Rename/remove non-HADOOP\_\*, etc from the shell scripts | Blocker | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12294](https://issues.apache.org/jira/browse/HADOOP-12294) | Throw an Exception when fs.permissions.umask-mode is misconfigured | Major | conf | Chang Li | Chang Li |
+| [HADOOP-10465](https://issues.apache.org/jira/browse/HADOOP-10465) | Fix use of generics within SortedMapWritable | Minor | . | Bertrand Dechoux | Bertrand Dechoux |
+| [HDFS-5165](https://issues.apache.org/jira/browse/HDFS-5165) | Remove the TotalFiles metrics | Minor | . | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-9047](https://issues.apache.org/jira/browse/HDFS-9047) | Retire libwebhdfs | Major | webhdfs | Allen Wittenauer | Haohui Mai |
+| [HADOOP-12651](https://issues.apache.org/jira/browse/HADOOP-12651) | Replace dev-support with wrappers to Yetus | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-6613](https://issues.apache.org/jira/browse/MAPREDUCE-6613) | Change mapreduce.jobhistory.jhist.format default from json to binary | Minor | . | Ray Chiang | Ray Chiang |
+| [HDFS-9395](https://issues.apache.org/jira/browse/HDFS-9395) | Make HDFS audit logging consistant | Major | . | Kihwal Lee | Kuhu Shukla |
+| [HADOOP-12552](https://issues.apache.org/jira/browse/HADOOP-12552) | Fix undeclared/unused dependency to httpclient | Minor | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-12850](https://issues.apache.org/jira/browse/HADOOP-12850) | pull shell code out of hadoop-dist | Major | build | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11792](https://issues.apache.org/jira/browse/HADOOP-11792) | Remove all of the CHANGES.txt files | Major | build | Allen Wittenauer | Andrew Wang |
+| [HADOOP-12857](https://issues.apache.org/jira/browse/HADOOP-12857) | Rework hadoop-tools | Major | build | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12967](https://issues.apache.org/jira/browse/HADOOP-12967) | Remove FileUtil#copyMerge | Major | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HADOOP-12811](https://issues.apache.org/jira/browse/HADOOP-12811) | Change kms server port number which conflicts with HMaster port number | Critical | kms | Yufeng Jiang | Xiao Chen |
+| [YARN-4784](https://issues.apache.org/jira/browse/YARN-4784) | Fairscheduler: defaultQueueSchedulingPolicy should not accept FIFO | Major | fairscheduler | Yufei Gu | Yufei Gu |
+| [HDFS-9427](https://issues.apache.org/jira/browse/HDFS-9427) | HDFS should not default to ephemeral ports | Critical | datanode, hdfs-client, namenode | Arpit Agarwal | Xiaobing Zhou |
+| [HADOOP-12563](https://issues.apache.org/jira/browse/HADOOP-12563) | Updated utility to create/modify token files | Major | . | Allen Wittenauer | Matthew Paduano |
+| [HADOOP-13045](https://issues.apache.org/jira/browse/HADOOP-13045) | hadoop\_add\_classpath is not working in .hadooprc | Major | . | Akira Ajisaka | Allen Wittenauer |
+| [MAPREDUCE-6526](https://issues.apache.org/jira/browse/MAPREDUCE-6526) | Remove usage of metrics v1 from hadoop-mapreduce | Blocker | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-12504](https://issues.apache.org/jira/browse/HADOOP-12504) | Remove metrics v1 | Blocker | metrics | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-10337](https://issues.apache.org/jira/browse/HDFS-10337) | OfflineEditsViewer stats option should print 0 instead of null for the count of operations | Minor | . | Akira Ajisaka | Yiqun Lin |
+| [HADOOP-11858](https://issues.apache.org/jira/browse/HADOOP-11858) | [JDK8] Set minimum version of Hadoop 3 to JDK 8 | Blocker | build | Robert Kanter | Robert Kanter |
+| [YARN-5035](https://issues.apache.org/jira/browse/YARN-5035) | FairScheduler: Adjust maxAssign dynamically when assignMultiple is turned on | Major | fairscheduler | Karthik Kambatla | Karthik Kambatla |
+| [HADOOP-13175](https://issues.apache.org/jira/browse/HADOOP-13175) | Remove hadoop-ant from hadoop-tools | Major | . | Chris Douglas | Chris Douglas |
+| [HADOOP-12892](https://issues.apache.org/jira/browse/HADOOP-12892) | fix/rewrite create-release | Blocker | build | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-9613](https://issues.apache.org/jira/browse/HADOOP-9613) | [JDK8] Update jersey version to latest 1.x release | Major | build | Timothy St. Clair | Tsuyoshi Ozawa |
+| [HADOOP-13209](https://issues.apache.org/jira/browse/HADOOP-13209) | replace slaves with workers | Major | scripts | John Smith | John Smith |
+| [HDFS-6434](https://issues.apache.org/jira/browse/HDFS-6434) | Default permission for creating file should be 644 for WebHdfs/HttpFS | Minor | webhdfs | Juan Yu | Wellington Chevreuil |
+| [HADOOP-12864](https://issues.apache.org/jira/browse/HADOOP-12864) | Remove bin/rcc script | Blocker | tools | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12709](https://issues.apache.org/jira/browse/HADOOP-12709) | Cut s3:// from trunk | Major | fs/s3 | Steve Loughran | Mingliang Liu |
+| [HADOOP-12064](https://issues.apache.org/jira/browse/HADOOP-12064) | [JDK8] Update guice version to 4.0 | Blocker | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa |
+| [HDFS-10548](https://issues.apache.org/jira/browse/HDFS-10548) | Remove the long deprecated BlockReaderRemote | Major | hdfs | Kai Zheng | Kai Zheng |
+| [HADOOP-13382](https://issues.apache.org/jira/browse/HADOOP-13382) | remove unneeded commons-httpclient dependencies from POM files in Hadoop and sub-projects | Major | build | Matt Foley | Matt Foley |
+| [HDFS-10689](https://issues.apache.org/jira/browse/HDFS-10689) | Hdfs dfs chmod should reset sticky bit permission when the bit is omitted in the octal mode | Minor | fs | Manoj Govindassamy | Manoj Govindassamy |
+| [HDFS-7933](https://issues.apache.org/jira/browse/HDFS-7933) | fsck should also report decommissioning replicas. | Major | namenode | Jitendra Nath Pandey | Xiaoyu Yao |
+| [HADOOP-12726](https://issues.apache.org/jira/browse/HADOOP-12726) | Unsupported FS operations should throw UnsupportedOperationException | Major | fs | Daniel Templeton | Daniel Templeton |
+
+
+### IMPORTANT ISSUES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-4785](https://issues.apache.org/jira/browse/YARN-4785) | inconsistent value type of the "type" field for LeafQueueInfo in response of RM REST API - cluster/scheduler | Major | webapp | Jayesh | Varun Vasudev |
+| [HADOOP-11393](https://issues.apache.org/jira/browse/HADOOP-11393) | Revert HADOOP\_PREFIX, go back to HADOOP\_HOME | Major | scripts, tracing | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12930](https://issues.apache.org/jira/browse/HADOOP-12930) | [Umbrella] Dynamic subcommands for hadoop shell scripts | Critical | scripts | Allen Wittenauer | Allen Wittenauer |
+
+
+### NEW FEATURES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [MAPREDUCE-4887](https://issues.apache.org/jira/browse/MAPREDUCE-4887) | Rehashing partitioner for better distribution | Major | . | Radim Kolar | Radim Kolar |
+| [HADOOP-11013](https://issues.apache.org/jira/browse/HADOOP-11013) | CLASSPATH handling should be consolidated, debuggable | Major | . | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-6590](https://issues.apache.org/jira/browse/HADOOP-6590) | Add a username check for hadoop sub-commands | Minor | scripts | Scott Chen | John Smith |
+| [HADOOP-11353](https://issues.apache.org/jira/browse/HADOOP-11353) | Add support for .hadooprc | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-9044](https://issues.apache.org/jira/browse/HADOOP-9044) | add FindClass main class to provide classpath checking of installations | Minor | util | Steve Loughran | Steve Loughran |
+| [HADOOP-11485](https://issues.apache.org/jira/browse/HADOOP-11485) | Pluggable shell integration | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-8934](https://issues.apache.org/jira/browse/HADOOP-8934) | Shell command ls should include sort options | Minor | fs | Jonathan Allen | Jonathan Allen |
+| [HADOOP-11554](https://issues.apache.org/jira/browse/HADOOP-11554) | Expose HadoopKerberosName as a hadoop subcommand | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11565](https://issues.apache.org/jira/browse/HADOOP-11565) | Add --slaves shell option | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-9477](https://issues.apache.org/jira/browse/HADOOP-9477) | Add posixGroups support for LDAP groups mapping service | Major | . | Kai Zheng | Dapeng Sun |
+| [HDFS-8009](https://issues.apache.org/jira/browse/HDFS-8009) | Signal congestion on the DataNode | Major | datanode | Haohui Mai | Haohui Mai |
+| [YARN-2901](https://issues.apache.org/jira/browse/YARN-2901) | Add errors and warning metrics page to RM, NM web UI | Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev |
+| [HADOOP-9642](https://issues.apache.org/jira/browse/HADOOP-9642) | Configuration to resolve environment variables via ${env.VARIABLE} references | Minor | conf, scripts | Steve Loughran | Kengo Seki |
+| [HDFS-7891](https://issues.apache.org/jira/browse/HDFS-7891) | A block placement policy with best rack failure tolerance | Minor | namenode | Walter Su | Walter Su |
+| [HADOOP-11843](https://issues.apache.org/jira/browse/HADOOP-11843) | Make setting up the build environment easier | Major | build | Niels Basjes | Niels Basjes |
+| [MAPREDUCE-6284](https://issues.apache.org/jira/browse/MAPREDUCE-6284) | Add Task Attempt State API to MapReduce Application Master REST API | Minor | . | Ryu Kobayashi | Ryu Kobayashi |
+| [HADOOP-10971](https://issues.apache.org/jira/browse/HADOOP-10971) | Add -C flag to make `hadoop fs -ls` print filenames only | Major | fs | Ryan Williams | Kengo Seki |
+| [MAPREDUCE-6364](https://issues.apache.org/jira/browse/MAPREDUCE-6364) | Add a "Kill" link to Task Attempts page | Minor | applicationmaster | Ryu Kobayashi | Ryu Kobayashi |
+| [MAPREDUCE-6304](https://issues.apache.org/jira/browse/MAPREDUCE-6304) | Specifying node labels when submitting MR jobs | Major | . | Jian Fang | Naganarasimha G R |
+| [HADOOP-7947](https://issues.apache.org/jira/browse/HADOOP-7947) | Validate XMLs if a relevant tool is available, when using scripts | Major | scripts | Harsh J | Kengo Seki |
+| [HDFS-8487](https://issues.apache.org/jira/browse/HDFS-8487) | Generalize BlockInfo in preparation of merging HDFS-7285 into trunk and branch-2 | Major | namenode | Zhe Zhang | Zhe Zhang |
+| [HDFS-8608](https://issues.apache.org/jira/browse/HDFS-8608) | Merge HDFS-7912 to trunk and branch-2 (track BlockInfo instead of Block in UnderReplicatedBlocks and PendingReplicationBlocks) | Major | . | Zhe Zhang | Zhe Zhang |
+| [HDFS-6440](https://issues.apache.org/jira/browse/HDFS-6440) | Support more than 2 NameNodes | Major | auto-failover, ha, namenode | Jesse Yates | Jesse Yates |
+| [HADOOP-5732](https://issues.apache.org/jira/browse/HADOOP-5732) | Add SFTP FileSystem | Minor | fs | ��igo Goiri | ramtin |
+| [HDFS-8622](https://issues.apache.org/jira/browse/HDFS-8622) | Implement GETCONTENTSUMMARY operation for WebImageViewer | Major | . | Jagadesh Kiran N | Jagadesh Kiran N |
+| [HDFS-8155](https://issues.apache.org/jira/browse/HDFS-8155) | Support OAuth2 in WebHDFS | Major | webhdfs | Jakob Homan | Jakob Homan |
+| [MAPREDUCE-6415](https://issues.apache.org/jira/browse/MAPREDUCE-6415) | Create a tool to combine aggregated logs into HAR files | Major | . | Robert Kanter | Robert Kanter |
+| [HADOOP-12360](https://issues.apache.org/jira/browse/HADOOP-12360) | Create StatsD metrics2 sink | Minor | metrics | Dave Marion | Dave Marion |
+| [HDFS-7285](https://issues.apache.org/jira/browse/HDFS-7285) | Erasure Coding Support inside HDFS | Major | . | Weihua Jiang | Zhe Zhang |
+| [HADOOP-10949](https://issues.apache.org/jira/browse/HADOOP-10949) | metrics2 sink plugin for Apache Kafka | Major | metrics | Babak Behzad | Babak Behzad |
+| [YARN-261](https://issues.apache.org/jira/browse/YARN-261) | Ability to fail AM attempts | Major | api | Jason Lowe | Rohith Sharma K S |
+| [HDFS-9184](https://issues.apache.org/jira/browse/HDFS-9184) | Logging HDFS operation's caller context into audit logs | Major | . | Mingliang Liu | Mingliang Liu |
+| [HDFS-9057](https://issues.apache.org/jira/browse/HDFS-9057) | allow/disallow snapshots via webhdfs | Major | webhdfs | Allen Wittenauer | Brahma Reddy Battula |
+| [YARN-4349](https://issues.apache.org/jira/browse/YARN-4349) | Support CallerContext in YARN | Major | . | Wangda Tan | Wangda Tan |
+| [HADOOP-12366](https://issues.apache.org/jira/browse/HADOOP-12366) | expose calculated paths | Major | . | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12321](https://issues.apache.org/jira/browse/HADOOP-12321) | Make JvmPauseMonitor an AbstractService | Major | . | Steve Loughran | Sunil G |
+| [YARN-3623](https://issues.apache.org/jira/browse/YARN-3623) | We should have a config to indicate the Timeline Service version | Major | timelineserver | Zhijie Shen | Xuan Gong |
+| [HADOOP-12657](https://issues.apache.org/jira/browse/HADOOP-12657) | Add a option to skip newline on empty files with getMerge -nl | Minor | . | Jan Filipiak | Kanaka Kumar Avvaru |
+| [YARN-3458](https://issues.apache.org/jira/browse/YARN-3458) | CPU resource monitoring in Windows | Minor | nodemanager | Inigo Goiri | Inigo Goiri |
+| [HADOOP-12691](https://issues.apache.org/jira/browse/HADOOP-12691) | Add CSRF Filter for REST APIs to Hadoop Common | Major | security | Larry McCay | Larry McCay |
+| [HADOOP-12635](https://issues.apache.org/jira/browse/HADOOP-12635) | Adding Append API support for WASB | Major | azure | Dushyanth | Dushyanth |
+| [HDFS-9525](https://issues.apache.org/jira/browse/HDFS-9525) | hadoop utilities need to support provided delegation tokens | Blocker | security | Allen Wittenauer | HeeSoo Kim |
+| [HADOOP-12702](https://issues.apache.org/jira/browse/HADOOP-12702) | Add an HDFS metrics sink | Major | metrics | Daniel Templeton | Daniel Templeton |
+| [HADOOP-12426](https://issues.apache.org/jira/browse/HADOOP-12426) | Add Entry point for Kerberos health check | Minor | security | Steve Loughran | Steve Loughran |
+| [HDFS-9244](https://issues.apache.org/jira/browse/HDFS-9244) | Support nested encryption zones | Major | encryption | Xiaoyu Yao | Zhe Zhang |
+| [HADOOP-12548](https://issues.apache.org/jira/browse/HADOOP-12548) | Read s3a creds from a Credential Provider | Major | fs/s3 | Allen Wittenauer | Larry McCay |
+| [HDFS-9711](https://issues.apache.org/jira/browse/HDFS-9711) | Integrate CSRF prevention filter in WebHDFS. | Major | datanode, namenode, webhdfs | Chris Nauroth | Chris Nauroth |
+| [HDFS-9835](https://issues.apache.org/jira/browse/HDFS-9835) | OIV: add ReverseXML processor which reconstructs an fsimage from an XML file | Major | tools | Colin P. McCabe | Colin P. McCabe |
+| [HDFS-9239](https://issues.apache.org/jira/browse/HDFS-9239) | DataNode Lifeline Protocol: an alternative protocol for reporting DataNode liveness | Major | datanode, namenode | Chris Nauroth | Chris Nauroth |
+| [HADOOP-12909](https://issues.apache.org/jira/browse/HADOOP-12909) | Change ipc.Client to support asynchronous calls | Major | ipc | Tsz Wo Nicholas Sze | Xiaobing Zhou |
+| [HDFS-9945](https://issues.apache.org/jira/browse/HDFS-9945) | Datanode command for evicting writers | Major | datanode | Kihwal Lee | Kihwal Lee |
+| [HADOOP-13008](https://issues.apache.org/jira/browse/HADOOP-13008) | Add XFS Filter for UIs to Hadoop Common | Major | security | Larry McCay | Larry McCay |
+| [HADOOP-13065](https://issues.apache.org/jira/browse/HADOOP-13065) | Add a new interface for retrieving FS and FC Statistics | Major | fs | Ram Venkatesh | Mingliang Liu |
+| [HADOOP-12723](https://issues.apache.org/jira/browse/HADOOP-12723) | S3A: Add ability to plug in any AWSCredentialsProvider | Major | fs/s3 | Steven K. Wong | Steven K. Wong |
+| [HADOOP-12847](https://issues.apache.org/jira/browse/HADOOP-12847) | hadoop daemonlog should support https and SPNEGO for Kerberized cluster | Major | security | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-13226](https://issues.apache.org/jira/browse/HADOOP-13226) | Support async call retry and failover | Major | io, ipc | Xiaobing Zhou | Tsz Wo Nicholas Sze |
+| [HADOOP-12537](https://issues.apache.org/jira/browse/HADOOP-12537) | S3A to support Amazon STS temporary credentials | Minor | fs/s3 | Sean Mackrory | Sean Mackrory |
+| [HADOOP-12666](https://issues.apache.org/jira/browse/HADOOP-12666) | Support Microsoft Azure Data Lake - as a file system in Hadoop | Major | fs, fs/azure, tools | Vishwajeet Dusane | Vishwajeet Dusane |
+| [HDFS-1312](https://issues.apache.org/jira/browse/HDFS-1312) | Re-balance disks within a Datanode | Major | datanode | Travis Crawford | Anu Engineer |
+| [YARN-2928](https://issues.apache.org/jira/browse/YARN-2928) | YARN Timeline Service v.2: alpha 1 | Critical | timelineserver | Sangjin Lee | Sangjin Lee |
+| [MAPREDUCE-6331](https://issues.apache.org/jira/browse/MAPREDUCE-6331) | [Umbrella] Make MapReduce work with Timeline Service Nextgen (YARN-2928) | Major | . | Vinod Kumar Vavilapalli | Sangjin Lee |
+| [HADOOP-12747](https://issues.apache.org/jira/browse/HADOOP-12747) | support wildcard in libjars argument | Major | util | Sangjin Lee | Sangjin Lee |
+| [HDFS-9804](https://issues.apache.org/jira/browse/HDFS-9804) | Allow long-running Balancer to login with keytab | Major | balancer & mover, security | Xiao Chen | Xiao Chen |
+| [MAPREDUCE-6690](https://issues.apache.org/jira/browse/MAPREDUCE-6690) | Limit the number of resources a single map reduce job can submit for localization | Major | . | Chris Trezzo | Chris Trezzo |
+| [MAPREDUCE-6578](https://issues.apache.org/jira/browse/MAPREDUCE-6578) | Add support for HDFS heterogeneous storage testing to TestDFSIO | Major | . | Wei Zhou | Wei Zhou |
+| [HADOOP-13396](https://issues.apache.org/jira/browse/HADOOP-13396) | Allow pluggable audit loggers in KMS | Major | kms | Xiao Chen | Xiao Chen |
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [MAPREDUCE-2944](https://issues.apache.org/jira/browse/MAPREDUCE-2944) | Improve checking of input for Api displayTasks() | Trivial | client | XieXianshan | XieXianshan |
+| [HADOOP-8017](https://issues.apache.org/jira/browse/HADOOP-8017) | Configure hadoop-main pom to get rid of M2E plugin execution not covered | Major | build | Eric Charles | Eric Charles |
+| [HADOOP-8078](https://issues.apache.org/jira/browse/HADOOP-8078) | Add capability to turn on security in unit tests. | Major | test | Jitendra Nath Pandey | Jaimin D Jetly |
+| [MAPREDUCE-3956](https://issues.apache.org/jira/browse/MAPREDUCE-3956) | Remove the use of the deprecated Syncable.sync() method | Minor | examples | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HADOOP-8147](https://issues.apache.org/jira/browse/HADOOP-8147) | test-patch should run tests with -fn to avoid masking test failures | Major | test | Robert Joseph Evans | Robert Joseph Evans |
+| [HADOOP-8290](https://issues.apache.org/jira/browse/HADOOP-8290) | Remove two remaining references to "hadoop.native.lib" oldprop | Trivial | test | Harsh J | Harsh J |
+| [HDFS-2743](https://issues.apache.org/jira/browse/HDFS-2743) | Streamline usage of bookkeeper journal manager | Major | ha | Ivan Kelly | Ivan Kelly |
+| [HADOOP-8312](https://issues.apache.org/jira/browse/HADOOP-8312) | testpatch.sh should provide a simpler way to see which warnings changed | Major | scripts | Robert Joseph Evans | Robert Joseph Evans |
+| [HDFS-3293](https://issues.apache.org/jira/browse/HDFS-3293) | Implement equals for journainfo class. | Minor | namenode | Hari Mankude | Hari Mankude |
+| [HADOOP-8375](https://issues.apache.org/jira/browse/HADOOP-8375) | test-patch should stop immediately once it has found compilation errors | Major | build | Tsz Wo Nicholas Sze | Robert Joseph Evans |
+| [HADOOP-8297](https://issues.apache.org/jira/browse/HADOOP-8297) | Writable javadocs don't carry default constructor | Trivial | documentation | Harsh J | Harsh J |
+| [HADOOP-7659](https://issues.apache.org/jira/browse/HADOOP-7659) | fs -getmerge isn't guaranteed to work well over non-HDFS filesystems | Minor | fs | Harsh J | Harsh J |
+| [MAPREDUCE-3302](https://issues.apache.org/jira/browse/MAPREDUCE-3302) | Remove the last dependency call from org.apache.hadoop.record package in MR. | Minor | client | Harsh J | Harsh J |
+| [MAPREDUCE-2384](https://issues.apache.org/jira/browse/MAPREDUCE-2384) | The job submitter should make sure to validate jobs before creation of necessary files | Major | job submission, test | Denny Ye | Harsh J |
+| [HDFS-3476](https://issues.apache.org/jira/browse/HDFS-3476) | Correct the default used in TestDFSClientRetries.busyTest() after HDFS-3462 | Minor | test | Harsh J | Harsh J |
+| [HDFS-3478](https://issues.apache.org/jira/browse/HDFS-3478) | Test quotas with Long.Max\_Value | Minor | namenode | Sujay Rau | Sujay Rau |
+| [MAPREDUCE-4371](https://issues.apache.org/jira/browse/MAPREDUCE-4371) | Check for cyclic dependencies in Jobcontrol job DAG | Major | mrv1 | madhukara phatak | madhukara phatak |
+| [HADOOP-8523](https://issues.apache.org/jira/browse/HADOOP-8523) | test-patch.sh doesn't validate patches before building | Minor | build | Jack Dintruff | Jack Dintruff |
+| [HADOOP-8521](https://issues.apache.org/jira/browse/HADOOP-8521) | Port StreamInputFormat to new Map Reduce API | Major | streaming | madhukara phatak | madhukara phatak |
+| [HADOOP-8593](https://issues.apache.org/jira/browse/HADOOP-8593) | add the missed @Override to methods in Metric/Metric2 package | Minor | metrics | Brandon Li | Brandon Li |
+| [HDFS-3768](https://issues.apache.org/jira/browse/HDFS-3768) | Exception in TestJettyHelper is incorrect | Minor | test | Jakob Homan | Eli Reisman |
+| [HDFS-3834](https://issues.apache.org/jira/browse/HDFS-3834) | Unused Command.java fields NAME, DESCRIPTION and USAGE | Minor | shell | Jing Zhao | Jing Zhao |
+| [HDFS-2580](https://issues.apache.org/jira/browse/HDFS-2580) | NameNode#main(...) can make use of GenericOptionsParser. | Minor | namenode | Harsh J | Harsh J |
+| [HADOOP-8588](https://issues.apache.org/jira/browse/HADOOP-8588) | SerializationFactory shouldn't throw a NullPointerException if the serializations list is empty | Minor | io | Harsh J | Sho Shimauchi |
+| [HADOOP-7930](https://issues.apache.org/jira/browse/HADOOP-7930) | Kerberos relogin interval in UserGroupInformation should be configurable | Major | security | Alejandro Abdelnur | Robert Kanter |
+| [HADOOP-8838](https://issues.apache.org/jira/browse/HADOOP-8838) | Colorize the test-patch output sent to JIRA | Trivial | build | Harsh J | Harsh J |
+| [MAPREDUCE-4686](https://issues.apache.org/jira/browse/MAPREDUCE-4686) | hadoop-mapreduce-client-core fails compilation in Eclipse due to missing Avro-generated classes | Major | build | Chris Nauroth | Chris Nauroth |
+| [HADOOP-8864](https://issues.apache.org/jira/browse/HADOOP-8864) | Addendum to HADOOP-8840: Add a coloring case for +0 results too. | Trivial | precommit | Harsh J | Harsh J |
+| [HDFS-3995](https://issues.apache.org/jira/browse/HDFS-3995) | Use DFSTestUtil.createFile() for file creation and writing in test cases | Minor | test | Jing Zhao | Jing Zhao |
+| [HADOOP-8920](https://issues.apache.org/jira/browse/HADOOP-8920) | Add more javadoc to metrics2 related classes | Minor | metrics | Suresh Srinivas | Suresh Srinivas |
+| [HADOOP-8776](https://issues.apache.org/jira/browse/HADOOP-8776) | Provide an option in test-patch that can enable / disable compiling native code | Minor | build | Hemanth Yamijala | Chris Nauroth |
+| [MAPREDUCE-4735](https://issues.apache.org/jira/browse/MAPREDUCE-4735) | Make arguments in TestDFSIO case insensitive | Trivial | test | Robert Kanter | Brandon Li |
+| [HADOOP-6616](https://issues.apache.org/jira/browse/HADOOP-6616) | Improve documentation for rack awareness | Major | documentation | Jeff Hammerbacher | Adam Faris |
+| [HADOOP-9075](https://issues.apache.org/jira/browse/HADOOP-9075) | FileContext#FSLinkResolver should be made static | Minor | . | Suresh Srinivas | Arpit Agarwal |
+| [HDFS-3358](https://issues.apache.org/jira/browse/HDFS-3358) | Specify explicitly that the NN UI status total is talking of persistent objects on heap. | Trivial | namenode | Harsh J | Harsh J |
+| [HADOOP-9277](https://issues.apache.org/jira/browse/HADOOP-9277) | Improve javadoc for FileContext | Minor | . | Andrew Wang | Andrew Wang |
+| [HADOOP-9394](https://issues.apache.org/jira/browse/HADOOP-9394) | Port findHangingTest.sh from HBase to Hadoop | Minor | . | Andrew Wang | Andrew Wang |
+| [HDFS-4665](https://issues.apache.org/jira/browse/HDFS-4665) | Move TestNetworkTopologyWithNodeGroup to Common | Minor | test | Luke Lu | Junping Du |
+| [MAPREDUCE-5232](https://issues.apache.org/jira/browse/MAPREDUCE-5232) | log classpath and other key properties on child JVM start | Major | mrv1, mrv2 | Sangjin Lee | Sangjin Lee |
+| [HDFS-5041](https://issues.apache.org/jira/browse/HDFS-5041) | Add the time of last heartbeat to dead server Web UI | Minor | . | Ted Yu | Ted Yu |
+| [HADOOP-10044](https://issues.apache.org/jira/browse/HADOOP-10044) | Improve the javadoc of rpc code | Minor | . | Sanjay Radia | Sanjay Radia |
+| [MAPREDUCE-5197](https://issues.apache.org/jira/browse/MAPREDUCE-5197) | Checkpoint Service: a library component to facilitate checkpoint of task state | Major | mrv2 | Carlo Curino | Carlo Curino |
+| [MAPREDUCE-5196](https://issues.apache.org/jira/browse/MAPREDUCE-5196) | CheckpointAMPreemptionPolicy implements preemption in MR AM via checkpointing | Major | mr-am, mrv2 | Carlo Curino | Carlo Curino |
+| [HDFS-6228](https://issues.apache.org/jira/browse/HDFS-6228) | comments typo fix for FsDatasetImpl.java | Trivial | . | yunjiong zhao | yunjiong zhao |
+| [HADOOP-8719](https://issues.apache.org/jira/browse/HADOOP-8719) | Workaround for kerberos-related log errors upon running any hadoop command on OSX | Trivial | scripts | Jianbin Wei | Jianbin Wei |
+| [HADOOP-10834](https://issues.apache.org/jira/browse/HADOOP-10834) | Typo in CredentialShell usage | Minor | security | Benoy Antony | Benoy Antony |
+| [MAPREDUCE-6013](https://issues.apache.org/jira/browse/MAPREDUCE-6013) | mapred version is missing | Major | scripts | Allen Wittenauer | Akira Ajisaka |
+| [MAPREDUCE-2841](https://issues.apache.org/jira/browse/MAPREDUCE-2841) | Task level native optimization | Major | task | Binglin Chang | Sean Zhong |
+| [YARN-2438](https://issues.apache.org/jira/browse/YARN-2438) | yarn-env.sh cleanup | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11092](https://issues.apache.org/jira/browse/HADOOP-11092) | hadoop shell commands should print usage if not given a class | Major | scripts | Bruno Mah� | Allen Wittenauer |
+| [HDFS-7278](https://issues.apache.org/jira/browse/HDFS-7278) | Add a command that allows sysadmins to manually trigger full block reports from a DN | Major | datanode | Colin P. McCabe | Colin P. McCabe |
+| [HADOOP-11025](https://issues.apache.org/jira/browse/HADOOP-11025) | hadoop-daemons.sh should just call hdfs directly | Major | scripts | Allen Wittenauer | Masatake Iwasaki |
+| [HADOOP-11150](https://issues.apache.org/jira/browse/HADOOP-11150) | hadoop command should show the reason on failure by invalid COMMAND or CLASSNAME | Minor | scripts | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-11208](https://issues.apache.org/jira/browse/HADOOP-11208) | Replace "daemon" with better name in script subcommands | Major | scripts | Yongjun Zhang | Allen Wittenauer |
+| [HADOOP-10926](https://issues.apache.org/jira/browse/HADOOP-10926) | Improve smart-apply-patch.sh to apply binary diffs | Major | . | Andrew Wang | Colin P. McCabe |
+| [HADOOP-11081](https://issues.apache.org/jira/browse/HADOOP-11081) | Document hadoop properties expected to be set by the shell code in \*-env.sh | Major | documentation, scripts | Allen Wittenauer | Allen Wittenauer |
+| [YARN-2472](https://issues.apache.org/jira/browse/YARN-2472) | yarn-daemons.sh should just call yarn directly | Major | scripts | Allen Wittenauer | Masatake Iwasaki |
+| [YARN-2301](https://issues.apache.org/jira/browse/YARN-2301) | Improve yarn container command | Major | . | Jian He | Naganarasimha G R |
+| [HADOOP-11352](https://issues.apache.org/jira/browse/HADOOP-11352) | Clean up test-patch.sh to disable "+1 contrib tests" | Major | build | Akira Ajisaka | Akira Ajisaka |
+| [YARN-2437](https://issues.apache.org/jira/browse/YARN-2437) | start-yarn.sh/stop-yarn should give info | Major | scripts | Allen Wittenauer | Varun Saxena |
+| [HDFS-7531](https://issues.apache.org/jira/browse/HDFS-7531) | Improve the concurrent access on FsVolumeList | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HADOOP-11058](https://issues.apache.org/jira/browse/HADOOP-11058) | Missing HADOOP\_CONF\_DIR generates strange results | Major | scripts | Allen Wittenauer | Masatake Iwasaki |
+| [HDFS-7579](https://issues.apache.org/jira/browse/HDFS-7579) | Improve log reporting during block report rpc failure | Minor | datanode | Charles Lamb | Charles Lamb |
+| [HDFS-7446](https://issues.apache.org/jira/browse/HDFS-7446) | HDFS inotify should have the ability to determine what txid it has read up to | Major | hdfs-client | Colin P. McCabe | Colin P. McCabe |
+| [HDFS-7182](https://issues.apache.org/jira/browse/HDFS-7182) | JMX metrics aren't accessible when NN is busy | Major | . | Ming Ma | Ming Ma |
+| [HADOOP-11466](https://issues.apache.org/jira/browse/HADOOP-11466) | FastByteComparisons: do not use UNSAFE\_COMPARER on the SPARC architecture because it is slower there | Minor | io, performance, util | Suman Somasundar | Suman Somasundar |
+| [HADOOP-11460](https://issues.apache.org/jira/browse/HADOOP-11460) | Deprecate shell vars | Major | scripts | Allen Wittenauer | John Smith |
+| [HADOOP-11346](https://issues.apache.org/jira/browse/HADOOP-11346) | Rewrite sls/rumen to use new shell framework | Major | scripts, tools | Allen Wittenauer | John Smith |
+| [HADOOP-7713](https://issues.apache.org/jira/browse/HADOOP-7713) | dfs -count -q should label output column | Trivial | . | Nigel Daley | Jonathan Allen |
+| [HADOOP-6964](https://issues.apache.org/jira/browse/HADOOP-6964) | Allow compact property description in xml | Major | conf | Owen O'Malley | Kengo Seki |
+| [HADOOP-11506](https://issues.apache.org/jira/browse/HADOOP-11506) | Configuration variable expansion regex expensive for long values | Major | conf | Dmitriy V. Ryaboy | Gera Shegalov |
+| [HDFS-7546](https://issues.apache.org/jira/browse/HDFS-7546) | Document, and set an accepting default for dfs.namenode.kerberos.principal.pattern | Minor | security | Harsh J | Harsh J |
+| [HDFS-7694](https://issues.apache.org/jira/browse/HDFS-7694) | FSDataInputStream should support "unbuffer" | Major | . | Colin P. McCabe | Colin P. McCabe |
+| [HDFS-7322](https://issues.apache.org/jira/browse/HDFS-7322) | deprecate sbin/hadoop-daemon.sh | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [YARN-2796](https://issues.apache.org/jira/browse/YARN-2796) | deprecate sbin/yarn-daemon.sh | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-6250](https://issues.apache.org/jira/browse/MAPREDUCE-6250) | deprecate sbin/mr-jobhistory-daemon.sh | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-6260](https://issues.apache.org/jira/browse/MAPREDUCE-6260) | Convert site documentation to markdown | Major | documentation | Allen Wittenauer | Masatake Iwasaki |
+| [HADOOP-11575](https://issues.apache.org/jira/browse/HADOOP-11575) | Daemon log documentation is misleading | Major | documentation | Jagadesh Kiran N | Naganarasimha G R |
+| [HADOOP-11596](https://issues.apache.org/jira/browse/HADOOP-11596) | Allow smart-apply-patch.sh to add new files in binary git patches | Major | . | Ravi Prakash | Ravi Prakash |
+| [HADOOP-11593](https://issues.apache.org/jira/browse/HADOOP-11593) | Convert site documentation from apt to markdown (stragglers) | Major | documentation | Allen Wittenauer | Masatake Iwasaki |
+| [YARN-3230](https://issues.apache.org/jira/browse/YARN-3230) | Clarify application states on the web UI | Major | . | Jian He | Jian He |
+| [HDFS-7460](https://issues.apache.org/jira/browse/HDFS-7460) | Rewrite httpfs to use new shell framework | Major | scripts | Allen Wittenauer | John Smith |
+| [YARN-3168](https://issues.apache.org/jira/browse/YARN-3168) | Convert site documentation from apt to markdown | Major | documentation | Allen Wittenauer | Gururaj Shetty |
+| [YARN-3199](https://issues.apache.org/jira/browse/YARN-3199) | Fair Scheduler documentation improvements | Minor | fairscheduler | Rohit Agarwal | Rohit Agarwal |
+| [MAPREDUCE-6267](https://issues.apache.org/jira/browse/MAPREDUCE-6267) | Refactor JobSubmitter#copyAndConfigureFiles into it's own class | Minor | . | Chris Trezzo | Chris Trezzo |
+| [YARN-3249](https://issues.apache.org/jira/browse/YARN-3249) | Add a "kill application" button to Resource Manager's Web UI | Minor | resourcemanager | Ryu Kobayashi | Ryu Kobayashi |
+| [HADOOP-11673](https://issues.apache.org/jira/browse/HADOOP-11673) | Use org.junit.Assume to skip tests instead of return | Minor | test | Akira Ajisaka | Brahma Reddy Battula |
+| [HADOOP-11692](https://issues.apache.org/jira/browse/HADOOP-11692) | Improve authentication failure WARN message to avoid user confusion | Major | ipc | Yongjun Zhang | Yongjun Zhang |
+| [HADOOP-11226](https://issues.apache.org/jira/browse/HADOOP-11226) | Add a configuration to set ipc.Client's traffic class with IPTOS\_LOWDELAY\|IPTOS\_RELIABILITY | Major | ipc | Gopal V | Gopal V |
+| [YARN-2280](https://issues.apache.org/jira/browse/YARN-2280) | Resource manager web service fields are not accessible | Trivial | resourcemanager | Krisztian Horvath | Krisztian Horvath |
+| [HADOOP-11711](https://issues.apache.org/jira/browse/HADOOP-11711) | Provide a default value for AES/CTR/NoPadding CryptoCodec classes | Minor | . | Andrew Wang | Andrew Wang |
+| [MAPREDUCE-4414](https://issues.apache.org/jira/browse/MAPREDUCE-4414) | Add main methods to JobConf and YarnConfiguration, for debug purposes | Major | client | Harsh J | Plamen Jeliazkov |
+| [MAPREDUCE-6105](https://issues.apache.org/jira/browse/MAPREDUCE-6105) | Inconsistent configuration in property mapreduce.reduce.shuffle.merge.percent | Trivial | . | Dongwook Kwon | Ray Chiang |
+| [HDFS-2360](https://issues.apache.org/jira/browse/HDFS-2360) | Ugly stacktrace when quota exceeds | Minor | hdfs-client | Rajit Saha | Harsh J |
+| [MAPREDUCE-6100](https://issues.apache.org/jira/browse/MAPREDUCE-6100) | replace "mapreduce.job.credentials.binary" with MRJobConfig.MAPREDUCE\_JOB\_CREDENTIALS\_BINARY for better readability. | Trivial | mrv2 | zhihai xu | zhihai xu |
+| [MAPREDUCE-5755](https://issues.apache.org/jira/browse/MAPREDUCE-5755) | MapTask.MapOutputBuffer#compare/swap should have @Override annotation | Trivial | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa |
+| [MAPREDUCE-4653](https://issues.apache.org/jira/browse/MAPREDUCE-4653) | TestRandomAlgorithm has an unused "import" statement | Trivial | contrib/gridmix | Amir Sanjar | Amir Sanjar |
+| [HADOOP-11659](https://issues.apache.org/jira/browse/HADOOP-11659) | o.a.h.fs.FileSystem.Cache#remove should use a single hash map lookup | Minor | fs | Gera Shegalov | Brahma Reddy Battula |
+| [HADOOP-11709](https://issues.apache.org/jira/browse/HADOOP-11709) | Time.NANOSECONDS\_PER\_MILLISECOND - use class-level final constant instead of method variable | Trivial | . | Ajith S | Ajith S |
+| [HDFS-7835](https://issues.apache.org/jira/browse/HDFS-7835) | make initial sleeptime in locateFollowingBlock configurable for DFSClient. | Major | hdfs-client | zhihai xu | zhihai xu |
+| [HDFS-7829](https://issues.apache.org/jira/browse/HDFS-7829) | Code clean up for LocatedBlock | Minor | . | Jing Zhao | Takanobu Asanuma |
+| [MAPREDUCE-6282](https://issues.apache.org/jira/browse/MAPREDUCE-6282) | Reuse historyFileAbsolute.getFileSystem in CompletedJob#loadFullHistoryData for code optimization. | Trivial | jobhistoryserver | zhihai xu | zhihai xu |
+| [HADOOP-11447](https://issues.apache.org/jira/browse/HADOOP-11447) | Add a more meaningful toString method to SampleStat and MutableStat | Minor | metrics | Karthik Kambatla | Karthik Kambatla |
+| [YARN-3350](https://issues.apache.org/jira/browse/YARN-3350) | YARN RackResolver spams logs with messages at info level | Major | . | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [MAPREDUCE-6239](https://issues.apache.org/jira/browse/MAPREDUCE-6239) | Consolidate TestJobConf classes in hadoop-mapreduce-client-jobclient and hadoop-mapreduce-client-core | Minor | client | Varun Saxena | Varun Saxena |
+| [MAPREDUCE-5190](https://issues.apache.org/jira/browse/MAPREDUCE-5190) | Unnecessary condition test in RandomSampler | Minor | mrv2 | Jingguo Yao | Jingguo Yao |
+| [MAPREDUCE-6287](https://issues.apache.org/jira/browse/MAPREDUCE-6287) | Deprecated methods in org.apache.hadoop.examples.Sort | Minor | examples | Chao Zhang | Chao Zhang |
+| [HADOOP-11737](https://issues.apache.org/jira/browse/HADOOP-11737) | mockito's version in hadoop-nfs\u2019 pom.xml shouldn't be specified | Minor | nfs | Kengo Seki | Kengo Seki |
+| [YARN-2868](https://issues.apache.org/jira/browse/YARN-2868) | FairScheduler: Metric for latency to allocate first container for an application | Major | . | Ray Chiang | Ray Chiang |
+| [HDFS-7875](https://issues.apache.org/jira/browse/HDFS-7875) | Improve log message when wrong value configured for dfs.datanode.failed.volumes.tolerated | Trivial | datanode | nijel | nijel |
+| [HDFS-7793](https://issues.apache.org/jira/browse/HDFS-7793) | Refactor DFSOutputStream separating DataStreamer out | Major | hdfs-client | Kai Zheng | Li Bo |
+| [HADOOP-11741](https://issues.apache.org/jira/browse/HADOOP-11741) | Add LOG.isDebugEnabled() guard for some LOG.debug() | Major | . | Walter Su | Walter Su |
+| [MAPREDUCE-579](https://issues.apache.org/jira/browse/MAPREDUCE-579) | Streaming "slowmatch" documentation | Trivial | contrib/streaming | Bo Adler | Harsh J |
+| [HDFS-7928](https://issues.apache.org/jira/browse/HDFS-7928) | Scanning blocks from disk during rolling upgrade startup takes a lot of time if disks are busy | Major | datanode | Rushabh S Shah | Rushabh S Shah |
+| [HADOOP-11524](https://issues.apache.org/jira/browse/HADOOP-11524) | hadoop\_do\_classpath\_subcommand throws a shellcheck warning | Minor | scripts | Allen Wittenauer | Chris Nauroth |
+| [HADOOP-11719](https://issues.apache.org/jira/browse/HADOOP-11719) | [Fsshell] Remove bin/hadoop reference from GenericOptionsParser default help text | Minor | scripts | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-8004](https://issues.apache.org/jira/browse/HDFS-8004) | Use KeyProviderCryptoExtension#warmUpEncryptedKeys when creating an encryption zone | Trivial | encryption | Andrew Wang | Andrew Wang |
+| [HDFS-7890](https://issues.apache.org/jira/browse/HDFS-7890) | Improve information on Top users for metrics in RollingWindowsManager and lower log level | Major | . | J.Andreina | J.Andreina |
+| [HDFS-4396](https://issues.apache.org/jira/browse/HDFS-4396) | Add START\_MSG/SHUTDOWN\_MSG for ZKFC | Major | auto-failover, ha, tools | Liang Xie | Liang Xie |
+| [HADOOP-11660](https://issues.apache.org/jira/browse/HADOOP-11660) | Add support for hardware crc of HDFS checksums on ARM aarch64 architecture | Minor | native | Edward Nevill | Edward Nevill |
+| [HDFS-7645](https://issues.apache.org/jira/browse/HDFS-7645) | Rolling upgrade is restoring blocks from trash multiple times | Major | datanode | Nathan Roberts | Keisuke Ogiwara |
+| [HDFS-3918](https://issues.apache.org/jira/browse/HDFS-3918) | EditLogTailer shouldn't log WARN when other node is in standby mode | Major | ha | Todd Lipcon | Todd Lipcon |
+| [HDFS-7944](https://issues.apache.org/jira/browse/HDFS-7944) | Minor cleanup of BlockPoolManager#getAllNamenodeThreads | Minor | . | Arpit Agarwal | Arpit Agarwal |
+| [YARN-3258](https://issues.apache.org/jira/browse/YARN-3258) | FairScheduler: Need to add more logging to investigate allocations | Minor | fairscheduler | Anubhav Dhoot | Anubhav Dhoot |
+| [HDFS-7671](https://issues.apache.org/jira/browse/HDFS-7671) | hdfs user guide should point to the common rack awareness doc | Major | documentation | Allen Wittenauer | Kai Sasaki |
+| [YARN-3412](https://issues.apache.org/jira/browse/YARN-3412) | RM tests should use MockRM where possible | Major | resourcemanager, test | Karthik Kambatla | Karthik Kambatla |
+| [YARN-3428](https://issues.apache.org/jira/browse/YARN-3428) | Debug log resources to be localized for a container | Trivial | nodemanager | Karthik Kambatla | Karthik Kambatla |
+| [YARN-3424](https://issues.apache.org/jira/browse/YARN-3424) | Change logs for ContainerMonitorImpl's resourse monitoring from info to debug | Major | nodemanager | Anubhav Dhoot | Anubhav Dhoot |
+| [YARN-3248](https://issues.apache.org/jira/browse/YARN-3248) | Display count of nodes blacklisted by apps in the web UI | Major | capacityscheduler, resourcemanager | Varun Vasudev | Varun Vasudev |
+| [HDFS-7978](https://issues.apache.org/jira/browse/HDFS-7978) | Add LOG.isDebugEnabled() guard for some LOG.debug(..) | Major | . | Walter Su | Walter Su |
+| [HDFS-8008](https://issues.apache.org/jira/browse/HDFS-8008) | Support client-side back off when the datanodes are congested | Major | hdfs-client | Haohui Mai | Haohui Mai |
+| [HDFS-7888](https://issues.apache.org/jira/browse/HDFS-7888) | Change DataStreamer/DFSOutputStream/DFSPacket for convenience of subclassing | Minor | hdfs-client | Li Bo | Li Bo |
+| [HDFS-8035](https://issues.apache.org/jira/browse/HDFS-8035) | Move checkBlocksProperlyReplicated() in FSNamesystem to BlockManager | Minor | namenode | Haohui Mai | Haohui Mai |
+| [HADOOP-9805](https://issues.apache.org/jira/browse/HADOOP-9805) | Refactor RawLocalFileSystem#rename for improved testability. | Minor | fs, test | Chris Nauroth | Jean-Pierre Matsumoto |
+| [HADOOP-11785](https://issues.apache.org/jira/browse/HADOOP-11785) | Reduce number of listStatus operation in distcp buildListing() | Minor | tools/distcp | Zoran Dimitrijevic | Zoran Dimitrijevic |
+| [HADOOP-11717](https://issues.apache.org/jira/browse/HADOOP-11717) | Add Redirecting WebSSO behavior with JWT Token in Hadoop Auth | Major | security | Larry McCay | Larry McCay |
+| [YARN-3294](https://issues.apache.org/jira/browse/YARN-3294) | Allow dumping of Capacity Scheduler debug logs via web UI for a fixed time period | Major | capacityscheduler | Varun Vasudev | Varun Vasudev |
+| [HDFS-8073](https://issues.apache.org/jira/browse/HDFS-8073) | Split BlockPlacementPolicyDefault.chooseTarget(..) so it can be easily overrided. | Trivial | namenode | Walter Su | Walter Su |
+| [HDFS-8076](https://issues.apache.org/jira/browse/HDFS-8076) | Code cleanup for DFSInputStream: use offset instead of LocatedBlock when possible | Major | . | Zhe Zhang | Zhe Zhang |
+| [HDFS-7979](https://issues.apache.org/jira/browse/HDFS-7979) | Initialize block report IDs with a random number | Minor | datanode | Andrew Wang | Andrew Wang |
+| [HDFS-8101](https://issues.apache.org/jira/browse/HDFS-8101) | DFSClient use of non-constant DFSConfigKeys pulls in WebHDFS classes at runtime | Minor | hdfs-client | Sean Busbey | Sean Busbey |
+| [YARN-3293](https://issues.apache.org/jira/browse/YARN-3293) | Track and display capacity scheduler health metrics in web UI | Major | capacityscheduler | Varun Vasudev | Varun Vasudev |
+| [MAPREDUCE-6291](https://issues.apache.org/jira/browse/MAPREDUCE-6291) | Correct mapred queue usage command | Minor | client | Brahma Reddy Battula | Brahma Reddy Battula |
+| [MAPREDUCE-6307](https://issues.apache.org/jira/browse/MAPREDUCE-6307) | Remove property mapreduce.tasktracker.taskmemorymanager.monitoringinterval | Minor | . | Akira Ajisaka | J.Andreina |
+| [YARN-3348](https://issues.apache.org/jira/browse/YARN-3348) | Add a 'yarn top' tool to help understand cluster usage | Major | resourcemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-3469](https://issues.apache.org/jira/browse/YARN-3469) | ZKRMStateStore: Avoid setting watches that are not required | Minor | . | Jun Gong | Jun Gong |
+| [HDFS-8117](https://issues.apache.org/jira/browse/HDFS-8117) | More accurate verification in SimulatedFSDataset: replace DEFAULT\_DATABYTE with patterned data | Major | . | Zhe Zhang | Zhe Zhang |
+| [HDFS-8144](https://issues.apache.org/jira/browse/HDFS-8144) | Split TestLazyPersistFiles into multiple tests | Major | test | Arpit Agarwal | Arpit Agarwal |
+| [YARN-3404](https://issues.apache.org/jira/browse/YARN-3404) | View the queue name to YARN Application page | Minor | . | Ryu Kobayashi | Ryu Kobayashi |
+| [YARN-3451](https://issues.apache.org/jira/browse/YARN-3451) | Add start time and Elapsed in ApplicationAttemptReport and display the same in RMAttemptBlock WebUI | Major | api, webapp | Rohith Sharma K S | Rohith Sharma K S |
+| [HDFS-7863](https://issues.apache.org/jira/browse/HDFS-7863) | Missing description of some methods and parameters in javadoc of FSDirDeleteOp | Minor | . | Yongjun Zhang | Brahma Reddy Battula |
+| [HDFS-8152](https://issues.apache.org/jira/browse/HDFS-8152) | Refactoring of lazy persist storage cases | Major | test | Arpit Agarwal | Arpit Agarwal |
+| [HDFS-8133](https://issues.apache.org/jira/browse/HDFS-8133) | Improve readability of deleted block check | Major | namenode | Daryn Sharp | Daryn Sharp |
+| [HADOOP-11812](https://issues.apache.org/jira/browse/HADOOP-11812) | Implement listLocatedStatus for ViewFileSystem to speed up split calculation | Blocker | fs | Gera Shegalov | Gera Shegalov |
+| [MAPREDUCE-6297](https://issues.apache.org/jira/browse/MAPREDUCE-6297) | Task Id of the failed task in diagnostics should link to the task page | Minor | jobhistoryserver | Siqi Li | Siqi Li |
+| [HADOOP-11827](https://issues.apache.org/jira/browse/HADOOP-11827) | Speed-up distcp buildListing() using threadpool | Major | tools/distcp | Zoran Dimitrijevic | Zoran Dimitrijevic |
+| [YARN-3494](https://issues.apache.org/jira/browse/YARN-3494) | Expose AM resource limit and usage in QueueMetrics | Major | . | Jian He | Rohith Sharma K S |
+| [YARN-3503](https://issues.apache.org/jira/browse/YARN-3503) | Expose disk utilization percentage and bad local and log dir counts on NM via JMX | Major | nodemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-3410](https://issues.apache.org/jira/browse/YARN-3410) | YARN admin should be able to remove individual application records from RMStateStore | Critical | resourcemanager, yarn | Wangda Tan | Rohith Sharma K S |
+| [HDFS-8215](https://issues.apache.org/jira/browse/HDFS-8215) | Refactor NamenodeFsck#check method | Minor | namenode | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-3511](https://issues.apache.org/jira/browse/YARN-3511) | Add errors and warnings page to ATS | Major | timelineserver | Varun Vasudev | Varun Vasudev |
+| [HDFS-8176](https://issues.apache.org/jira/browse/HDFS-8176) | Record from/to snapshots in audit log for snapshot diff report | Minor | snapshots | J.Andreina | J.Andreina |
+| [YARN-3406](https://issues.apache.org/jira/browse/YARN-3406) | Display count of running containers in the RM's Web UI | Minor | . | Ryu Kobayashi | Ryu Kobayashi |
+| [HADOOP-11357](https://issues.apache.org/jira/browse/HADOOP-11357) | Print information of the build enviornment in test-patch.sh | Minor | scripts | Haohui Mai | Allen Wittenauer |
+| [HADOOP-11857](https://issues.apache.org/jira/browse/HADOOP-11857) | Fix CommandFormat#commandFormat java doc annotation | Trivial | . | Xiaoyu Yao | J.Andreina |
+| [MAPREDUCE-6057](https://issues.apache.org/jira/browse/MAPREDUCE-6057) | Remove obsolete entries from mapred-default.xml | Minor | . | Ray Chiang | Ray Chiang |
+| [HDFS-8204](https://issues.apache.org/jira/browse/HDFS-8204) | Mover/Balancer should not schedule two replicas to the same DN | Minor | balancer & mover | Walter Su | Walter Su |
+| [HDFS-8280](https://issues.apache.org/jira/browse/HDFS-8280) | Code Cleanup in DFSInputStream | Minor | . | Jing Zhao | Jing Zhao |
+| [HDFS-8283](https://issues.apache.org/jira/browse/HDFS-8283) | DataStreamer cleanup and some minor improvement | Minor | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HDFS-5574](https://issues.apache.org/jira/browse/HDFS-5574) | Remove buffer copy in BlockReader.skip | Trivial | . | Binglin Chang | Binglin Chang |
+| [HDFS-7770](https://issues.apache.org/jira/browse/HDFS-7770) | Need document for storage type label of data node storage locations under dfs.datanode.data.dir | Major | documentation | Xiaoyu Yao | Xiaoyu Yao |
+| [HDFS-8200](https://issues.apache.org/jira/browse/HDFS-8200) | Refactor FSDirStatAndListingOp | Major | . | Haohui Mai | Haohui Mai |
+| [YARN-3363](https://issues.apache.org/jira/browse/YARN-3363) | add localization and container launch time to ContainerMetrics at NM to show these timing information for each active container. | Major | nodemanager | zhihai xu | zhihai xu |
+| [HDFS-7397](https://issues.apache.org/jira/browse/HDFS-7397) | Add more detail to the documentation for the conf key "dfs.client.read.shortcircuit.streams.cache.size" | Minor | hdfs-client | Tsz Wo Nicholas Sze | Brahma Reddy Battula |
+| [YARN-2980](https://issues.apache.org/jira/browse/YARN-2980) | Move health check script related functionality to hadoop-common | Blocker | . | Ming Ma | Varun Saxena |
+| [HADOOP-11911](https://issues.apache.org/jira/browse/HADOOP-11911) | test-patch should allow configuration of default branch | Minor | . | Sean Busbey | Sean Busbey |
+| [HDFS-7758](https://issues.apache.org/jira/browse/HDFS-7758) | Retire FsDatasetSpi#getVolumes() and use FsDatasetSpi#getVolumeRefs() instead | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [MAPREDUCE-6192](https://issues.apache.org/jira/browse/MAPREDUCE-6192) | Create unit test to automatically compare MR related classes and mapred-default.xml | Minor | . | Ray Chiang | Ray Chiang |
+| [HADOOP-11813](https://issues.apache.org/jira/browse/HADOOP-11813) | releasedocmaker.py should use today's date instead of unreleased | Minor | build | Allen Wittenauer | Darrell Taylor |
+| [YARN-3491](https://issues.apache.org/jira/browse/YARN-3491) | PublicLocalizer#addResource is too slow. | Critical | nodemanager | zhihai xu | zhihai xu |
+| [MAPREDUCE-6279](https://issues.apache.org/jira/browse/MAPREDUCE-6279) | AM should explicity exit JVM after all services have stopped | Major | . | Jason Lowe | Eric Payne |
+| [HDFS-8108](https://issues.apache.org/jira/browse/HDFS-8108) | Fsck should provide the info on mandatory option to be used along with "-blocks , -locations and -racks" | Trivial | documentation | J.Andreina | J.Andreina |
+| [HDFS-8207](https://issues.apache.org/jira/browse/HDFS-8207) | Improper log message when blockreport interval compared with initial delay | Minor | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [MAPREDUCE-6079](https://issues.apache.org/jira/browse/MAPREDUCE-6079) | Rename JobImpl#username to reporterUserName | Major | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa |
+| [HDFS-8209](https://issues.apache.org/jira/browse/HDFS-8209) | Support different number of datanode directories in MiniDFSCluster. | Minor | test | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [YARN-3169](https://issues.apache.org/jira/browse/YARN-3169) | Drop YARN's overview document | Major | documentation | Allen Wittenauer | Brahma Reddy Battula |
+| [YARN-2784](https://issues.apache.org/jira/browse/YARN-2784) | Make POM project names consistent | Minor | build | Rohith Sharma K S | Rohith Sharma K S |
+| [MAPREDUCE-5981](https://issues.apache.org/jira/browse/MAPREDUCE-5981) | Log levels of certain MR logs can be changed to DEBUG | Major | . | Varun Saxena | Varun Saxena |
+| [HADOOP-6842](https://issues.apache.org/jira/browse/HADOOP-6842) | "hadoop fs -text" does not give a useful text representation of MapWritable objects | Major | io | Steven K. Wong | Akira Ajisaka |
+| [YARN-20](https://issues.apache.org/jira/browse/YARN-20) | More information for "yarn.resourcemanager.webapp.address" in yarn-default.xml | Trivial | documentation, resourcemanager | Nemon Lou | Bartosz \u0141ugowski |
+| [HDFS-5640](https://issues.apache.org/jira/browse/HDFS-5640) | Add snapshot methods to FileContext. | Major | hdfs-client, snapshots | Chris Nauroth | Rakesh R |
+| [HDFS-8284](https://issues.apache.org/jira/browse/HDFS-8284) | Update documentation about how to use HTrace with HDFS | Major | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-7433](https://issues.apache.org/jira/browse/HDFS-7433) | Optimize performance of DatanodeManager's node map | Critical | namenode | Daryn Sharp | Daryn Sharp |
+| [HADOOP-11590](https://issues.apache.org/jira/browse/HADOOP-11590) | Update sbin commands and documentation to use new --slaves option | Blocker | documentation, scripts | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-5248](https://issues.apache.org/jira/browse/MAPREDUCE-5248) | Let NNBenchWithoutMR specify the replication factor for its test | Minor | client, test | Erik Paulson | Erik Paulson |
+| [HADOOP-9737](https://issues.apache.org/jira/browse/HADOOP-9737) | JarFinder#getJar should delete the jar file upon destruction of the JVM | Major | util | Esteban Gutierrez | Jean-Baptiste Onofr� |
+| [YARN-1050](https://issues.apache.org/jira/browse/YARN-1050) | Document the Fair Scheduler REST API | Major | documentation, fairscheduler | Sandy Ryza | Kenji Kikushima |
+| [YARN-3271](https://issues.apache.org/jira/browse/YARN-3271) | FairScheduler: Move tests related to max-runnable-apps from TestFairScheduler to TestAppRunnability | Major | . | Karthik Kambatla | nijel |
+| [YARN-2206](https://issues.apache.org/jira/browse/YARN-2206) | Update document for applications REST API response examples | Minor | documentation | Kenji Kikushima | Brahma Reddy Battula |
+| [HDFS-6757](https://issues.apache.org/jira/browse/HDFS-6757) | Simplify lease manager with INodeID | Major | . | Haohui Mai | Haohui Mai |
+| [HDFS-8327](https://issues.apache.org/jira/browse/HDFS-8327) | Simplify quota calculations for snapshots and truncate | Major | . | Haohui Mai | Haohui Mai |
+| [YARN-1287](https://issues.apache.org/jira/browse/YARN-1287) | Consolidate MockClocks | Major | . | Sandy Ryza | Sebastian Wong |
+| [HDFS-8357](https://issues.apache.org/jira/browse/HDFS-8357) | Consolidate parameters of INode.CleanSubtree() into a parameter objects. | Major | . | Haohui Mai | Li Lu |
+| [HADOOP-11950](https://issues.apache.org/jira/browse/HADOOP-11950) | Add cli option to test-patch to set the project-under-test | Minor | . | Sean Busbey | Sean Busbey |
+| [HADOOP-11948](https://issues.apache.org/jira/browse/HADOOP-11948) | test-patch's issue matching regex should be configurable. | Major | . | Sean Busbey | Sean Busbey |
+| [MAPREDUCE-5465](https://issues.apache.org/jira/browse/MAPREDUCE-5465) | Tasks are often killed before they exit on their own | Major | mr-am, mrv2 | Radim Kolar | Ming Ma |
+| [YARN-3513](https://issues.apache.org/jira/browse/YARN-3513) | Remove unused variables in ContainersMonitorImpl and add debug log for overall resource usage by all containers | Trivial | nodemanager | Naganarasimha G R | Naganarasimha G R |
+| [HDFS-8255](https://issues.apache.org/jira/browse/HDFS-8255) | Rename getBlockReplication to getPreferredBlockReplication | Major | . | Zhe Zhang | Zhe Zhang |
+| [YARN-3613](https://issues.apache.org/jira/browse/YARN-3613) | TestContainerManagerSecurity should init and start Yarn cluster in setup instead of individual methods | Minor | test | Karthik Kambatla | nijel |
+| [HDFS-6184](https://issues.apache.org/jira/browse/HDFS-6184) | Capture NN's thread dump when it fails over | Major | namenode | Ming Ma | Ming Ma |
+| [YARN-3539](https://issues.apache.org/jira/browse/YARN-3539) | Compatibility doc to state that ATS v1 is a stable REST API | Major | documentation | Steve Loughran | Steve Loughran |
+| [HADOOP-9723](https://issues.apache.org/jira/browse/HADOOP-9723) | Improve error message when hadoop archive output path already exists | Trivial | . | Stephen Chu | Yongjun Zhang |
+| [HADOOP-10993](https://issues.apache.org/jira/browse/HADOOP-10993) | Dump java command line to \*.out file | Minor | . | Allen Wittenauer | Kengo Seki |
+| [HADOOP-11713](https://issues.apache.org/jira/browse/HADOOP-11713) | ViewFileSystem should support snapshot methods. | Major | fs | Chris Nauroth | Rakesh R |
+| [HDFS-8350](https://issues.apache.org/jira/browse/HDFS-8350) | Remove old webhdfs.xml and other outdated documentation stuff | Major | documentation | Akira Ajisaka | Brahma Reddy Battula |
+| [HADOOP-11960](https://issues.apache.org/jira/browse/HADOOP-11960) | Enable Azure-Storage Client Side logging. | Major | tools | Dushyanth | Dushyanth |
+| [HDFS-6888](https://issues.apache.org/jira/browse/HDFS-6888) | Allow selectively audit logging ops | Major | . | Kihwal Lee | Chen He |
+| [HDFS-8397](https://issues.apache.org/jira/browse/HDFS-8397) | Refactor the error handling code in DataStreamer | Minor | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HDFS-8394](https://issues.apache.org/jira/browse/HDFS-8394) | Move getAdditionalBlock() and related functionalities into a separate class | Major | . | Haohui Mai | Haohui Mai |
+| [HADOOP-11939](https://issues.apache.org/jira/browse/HADOOP-11939) | Deprecate DistCpV1 and Logalyzer | Major | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-4185](https://issues.apache.org/jira/browse/HDFS-4185) | Add a metric for number of active leases | Major | namenode | Kihwal Lee | Rakesh R |
+| [HADOOP-1540](https://issues.apache.org/jira/browse/HADOOP-1540) | Support file exclusion list in distcp | Minor | util | Senthil Subramanian | Rich Haase |
+| [HADOOP-11103](https://issues.apache.org/jira/browse/HADOOP-11103) | Clean up RemoteException | Trivial | ipc | Sean Busbey | Sean Busbey |
+| [HDFS-8131](https://issues.apache.org/jira/browse/HDFS-8131) | Implement a space balanced block placement policy | Minor | namenode | Liu Shaohui | Liu Shaohui |
+| [HADOOP-11970](https://issues.apache.org/jira/browse/HADOOP-11970) | Replace uses of ThreadLocal\<Random\> with JDK7 ThreadLocalRandom | Major | . | Sean Busbey | Sean Busbey |
+| [HADOOP-11995](https://issues.apache.org/jira/browse/HADOOP-11995) | Make jetty version configurable from the maven command line | Trivial | build, ld | sriharsha devineni | sriharsha devineni |
+| [HDFS-4383](https://issues.apache.org/jira/browse/HDFS-4383) | Document the lease limits | Minor | . | Eli Collins | Arshad Mohammad |
+| [HADOOP-10366](https://issues.apache.org/jira/browse/HADOOP-10366) | Add whitespaces between the classes for values in core-default.xml to fit better in browser | Minor | documentation | Chengwei Yang | Kanaka Kumar Avvaru |
+| [HADOOP-11594](https://issues.apache.org/jira/browse/HADOOP-11594) | Improve the readability of site index of documentation | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-160](https://issues.apache.org/jira/browse/YARN-160) | nodemanagers should obtain cpu/memory values from underlying OS | Major | nodemanager | Alejandro Abdelnur | Varun Vasudev |
+| [HADOOP-11242](https://issues.apache.org/jira/browse/HADOOP-11242) | Record the time of calling in tracing span of IPC server | Minor | ipc | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-5033](https://issues.apache.org/jira/browse/HDFS-5033) | Bad error message for fs -put/copyFromLocal if user doesn't have permissions to read the source | Minor | . | Karthik Kambatla | Darrell Taylor |
+| [YARN-3722](https://issues.apache.org/jira/browse/YARN-3722) | Merge multiple TestWebAppUtils into o.a.h.yarn.webapp.util.TestWebAppUtils | Minor | test | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-11142](https://issues.apache.org/jira/browse/HADOOP-11142) | Remove hdfs dfs reference from file system shell documentation | Minor | documentation | Jonathan Allen | Kengo Seki |
+| [HADOOP-11894](https://issues.apache.org/jira/browse/HADOOP-11894) | Bump the version of Apache HTrace to 3.2.0-incubating | Major | . | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-3489](https://issues.apache.org/jira/browse/YARN-3489) | RMServerUtils.validateResourceRequests should only obtain queue info once | Major | resourcemanager | Jason Lowe | Varun Saxena |
+| [HDFS-8443](https://issues.apache.org/jira/browse/HDFS-8443) | Document dfs.namenode.service.handler.count in hdfs-site.xml | Major | documentation | Akira Ajisaka | J.Andreina |
+| [YARN-3547](https://issues.apache.org/jira/browse/YARN-3547) | FairScheduler: Apps that have no resource demand should not participate scheduling | Major | fairscheduler | Xianyin Xin | Xianyin Xin |
+| [YARN-3713](https://issues.apache.org/jira/browse/YARN-3713) | Remove duplicate function call storeContainerDiagnostics in ContainerDiagnosticsUpdateTransition | Minor | nodemanager | zhihai xu | zhihai xu |
+| [HADOOP-12043](https://issues.apache.org/jira/browse/HADOOP-12043) | Display warning if defaultFs is not set when running fs commands. | Minor | fs | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [YARN-3467](https://issues.apache.org/jira/browse/YARN-3467) | Expose allocatedMB, allocatedVCores, and runningContainers metrics on running Applications in RM Web UI | Minor | webapp, yarn | Anthony Rojas | Anubhav Dhoot |
+| [HDFS-8490](https://issues.apache.org/jira/browse/HDFS-8490) | Typo in trace enabled log in ExceptionHandler of WebHDFS | Trivial | webhdfs | Jakob Homan | Archana T |
+| [HDFS-8521](https://issues.apache.org/jira/browse/HDFS-8521) | Add @VisibleForTesting annotation to {{BlockPoolSlice#selectReplicaToDelete}} | Trivial | . | Colin P. McCabe | Colin P. McCabe |
+| [MAPREDUCE-6174](https://issues.apache.org/jira/browse/MAPREDUCE-6174) | Combine common stream code into parent class for InMemoryMapOutput and OnDiskMapOutput. | Major | mrv2 | Eric Payne | Eric Payne |
+| [HDFS-8532](https://issues.apache.org/jira/browse/HDFS-8532) | Make the visibility of DFSOutputStream#streamer member variable to private | Trivial | . | Rakesh R | Rakesh R |
+| [HDFS-8535](https://issues.apache.org/jira/browse/HDFS-8535) | Clarify that dfs usage in dfsadmin -report output includes all block replicas. | Minor | documentation | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [MAPREDUCE-6383](https://issues.apache.org/jira/browse/MAPREDUCE-6383) | Pi job (QuasiMonteCarlo) should not try to read the results file if its job fails | Major | examples | Harsh J | Harsh J |
+| [YARN-3259](https://issues.apache.org/jira/browse/YARN-3259) | FairScheduler: Trigger fairShare updates on node events | Major | fairscheduler | Anubhav Dhoot | Anubhav Dhoot |
+| [HADOOP-12059](https://issues.apache.org/jira/browse/HADOOP-12059) | S3Credentials should support use of CredentialProvider | Major | fs/s3 | Sean Busbey | Sean Busbey |
+| [MAPREDUCE-6354](https://issues.apache.org/jira/browse/MAPREDUCE-6354) | ShuffleHandler should be able to log shuffle connections | Major | . | Chang Li | Chang Li |
+| [HADOOP-12055](https://issues.apache.org/jira/browse/HADOOP-12055) | Deprecate usage of NativeIO#link | Major | native | Andrew Wang | Andrew Wang |
+| [HDFS-8432](https://issues.apache.org/jira/browse/HDFS-8432) | Introduce a minimum compatible layout version to allow downgrade in more rolling upgrade use cases. | Major | namenode, rolling upgrades | Chris Nauroth | Chris Nauroth |
+| [HDFS-8116](https://issues.apache.org/jira/browse/HDFS-8116) | Cleanup uncessary if LOG.isDebugEnabled() from RollingWindowManager | Trivial | namenode | Xiaoyu Yao | Brahma Reddy Battula |
+| [YARN-2716](https://issues.apache.org/jira/browse/YARN-2716) | Refactor ZKRMStateStore retry code with Apache Curator | Major | . | Jian He | Karthik Kambatla |
+| [HDFS-8553](https://issues.apache.org/jira/browse/HDFS-8553) | Document hdfs class path options | Major | documentation | Xiaoyu Yao | Brahma Reddy Battula |
+| [YARN-3786](https://issues.apache.org/jira/browse/YARN-3786) | Document yarn class path options | Major | documentation | Brahma Reddy Battula | Brahma Reddy Battula |
+| [MAPREDUCE-6392](https://issues.apache.org/jira/browse/MAPREDUCE-6392) | Document mapred class path options | Major | documentation | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-8549](https://issues.apache.org/jira/browse/HDFS-8549) | Abort the balancer if an upgrade is in progress | Major | balancer & mover | Andrew Wang | Andrew Wang |
+| [HDFS-8573](https://issues.apache.org/jira/browse/HDFS-8573) | Move creation of restartMeta file logic from BlockReceiver to ReplicaInPipeline | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HADOOP-11971](https://issues.apache.org/jira/browse/HADOOP-11971) | Move test utilities for tracing from hadoop-hdfs to hadoop-common | Minor | tracing | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-8361](https://issues.apache.org/jira/browse/HDFS-8361) | Choose SSD over DISK in block placement | Major | namenode | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [YARN-3789](https://issues.apache.org/jira/browse/YARN-3789) | Improve logs for LeafQueue#activateApplications() | Minor | resourcemanager | Bibin A Chundatt | Bibin A Chundatt |
+| [HDFS-8606](https://issues.apache.org/jira/browse/HDFS-8606) | Cleanup DFSOutputStream by removing unwanted changes | Minor | hdfs-client | Rakesh R | Rakesh R |
+| [YARN-3148](https://issues.apache.org/jira/browse/YARN-3148) | Allow CORS related headers to passthrough in WebAppProxyServlet | Major | . | Prakash Ramachandran | Varun Saxena |
+| [HDFS-8589](https://issues.apache.org/jira/browse/HDFS-8589) | Fix unused imports in BPServiceActor and BlockReportLeaseManager | Trivial | . | Colin P. McCabe | Colin P. McCabe |
+| [HADOOP-7139](https://issues.apache.org/jira/browse/HADOOP-7139) | Allow appending to existing SequenceFiles | Major | io | Stephen Rose | Kanaka Kumar Avvaru |
+| [HDFS-8605](https://issues.apache.org/jira/browse/HDFS-8605) | Merge Refactor of DFSOutputStream from HDFS-7285 branch | Major | . | Vinayakumar B | Vinayakumar B |
+| [MAPREDUCE-6395](https://issues.apache.org/jira/browse/MAPREDUCE-6395) | Improve the commit failure messages in MRAppMaster recovery | Major | applicationmaster | Gera Shegalov | Brahma Reddy Battula |
+| [HDFS-8582](https://issues.apache.org/jira/browse/HDFS-8582) | Support getting a list of reconfigurable config properties and do not generate spurious reconfig warnings | Minor | datanode, hdfs-client | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [MAPREDUCE-6316](https://issues.apache.org/jira/browse/MAPREDUCE-6316) | Task Attempt List entries should link to the task overview | Major | . | Siqi Li | Siqi Li |
+| [MAPREDUCE-6305](https://issues.apache.org/jira/browse/MAPREDUCE-6305) | AM/Task log page should be able to link back to the job | Major | . | Siqi Li | Siqi Li |
+| [YARN-3834](https://issues.apache.org/jira/browse/YARN-3834) | Scrub debug logging of tokens during resource localization. | Major | nodemanager | Chris Nauroth | Chris Nauroth |
+| [MAPREDUCE-6408](https://issues.apache.org/jira/browse/MAPREDUCE-6408) | Queue name and user name should be printed on the job page | Major | applicationmaster | Siqi Li | Siqi Li |
+| [HDFS-8639](https://issues.apache.org/jira/browse/HDFS-8639) | Option for HTTP port of NameNode by MiniDFSClusterManager | Minor | test | Kai Sasaki | Kai Sasaki |
+| [YARN-3360](https://issues.apache.org/jira/browse/YARN-3360) | Add JMX metrics to TimelineDataManager | Major | timelineserver | Jason Lowe | Jason Lowe |
+| [HADOOP-12049](https://issues.apache.org/jira/browse/HADOOP-12049) | Control http authentication cookie persistence via configuration | Major | security | Benoy Antony | Huizhi Lu |
+| [HDFS-8462](https://issues.apache.org/jira/browse/HDFS-8462) | Implement GETXATTRS and LISTXATTRS operations for WebImageViewer | Major | . | Akira Ajisaka | Jagadesh Kiran N |
+| [HDFS-8640](https://issues.apache.org/jira/browse/HDFS-8640) | Make reserved RBW space visible through JMX | Major | . | Kanaka Kumar Avvaru | Kanaka Kumar Avvaru |
+| [HDFS-8546](https://issues.apache.org/jira/browse/HDFS-8546) | Use try with resources in DataStorage and Storage | Minor | datanode | Andrew Wang | Andrew Wang |
+| [HADOOP-11203](https://issues.apache.org/jira/browse/HADOOP-11203) | Allow ditscp to accept bandwitdh in fraction MegaBytes | Major | tools/distcp | Raju Bairishetti | Raju Bairishetti |
+| [HADOOP-11807](https://issues.apache.org/jira/browse/HADOOP-11807) | add a lint mode to releasedocmaker | Minor | build, documentation, yetus | Allen Wittenauer | ramtin |
+| [HDFS-8653](https://issues.apache.org/jira/browse/HDFS-8653) | Code cleanup for DatanodeManager, DatanodeDescriptor and DatanodeStorageInfo | Major | . | Zhe Zhang | Zhe Zhang |
+| [HDFS-8659](https://issues.apache.org/jira/browse/HDFS-8659) | Block scanner INFO message is spamming logs | Major | datanode | Yongjun Zhang | Yongjun Zhang |
+| [HADOOP-12149](https://issues.apache.org/jira/browse/HADOOP-12149) | copy all of test-patch BINDIR prior to re-exec | Blocker | yetus | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-6384](https://issues.apache.org/jira/browse/MAPREDUCE-6384) | Add the last reporting reducer info for too many fetch failure diagnostics | Major | . | Chang Li | Chang Li |
+| [HADOOP-12158](https://issues.apache.org/jira/browse/HADOOP-12158) | Improve error message in TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not installed | Trivial | test | Andrew Wang | Andrew Wang |
+| [HADOOP-12172](https://issues.apache.org/jira/browse/HADOOP-12172) | FsShell mkdir -p makes an unnecessary check for the existence of the parent. | Minor | fs | Chris Nauroth | Chris Nauroth |
+| [HDFS-8703](https://issues.apache.org/jira/browse/HDFS-8703) | Merge refactor of DFSInputStream from ErasureCoding branch | Major | . | Vinayakumar B | Vinayakumar B |
+| [HDFS-8709](https://issues.apache.org/jira/browse/HDFS-8709) | Clarify automatic sync in FSEditLog#logEdit | Minor | . | Andrew Wang | Andrew Wang |
+| [HADOOP-12045](https://issues.apache.org/jira/browse/HADOOP-12045) | Enable LocalFileSystem#setTimes to change atime | Minor | fs | Kazuho Fujii | Kazuho Fujii |
+| [HADOOP-12185](https://issues.apache.org/jira/browse/HADOOP-12185) | NetworkTopology is not efficient adding/getting/removing nodes | Major | . | Inigo Goiri | Inigo Goiri |
+| [HADOOP-12135](https://issues.apache.org/jira/browse/HADOOP-12135) | cleanup releasedocmaker | Major | yetus | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12195](https://issues.apache.org/jira/browse/HADOOP-12195) | Add annotation to package-info.java file to workaround MCOMPILER-205 | Trivial | . | Andrew Wang | Andrew Wang |
+| [HADOOP-12193](https://issues.apache.org/jira/browse/HADOOP-12193) | Rename Touchz.java to Touch.java | Trivial | . | Andrew Wang | Andrew Wang |
+| [HDFS-8711](https://issues.apache.org/jira/browse/HDFS-8711) | setSpaceQuota command should print the available storage type when input storage type is wrong | Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-8712](https://issues.apache.org/jira/browse/HDFS-8712) | Remove "public" and "abstract" modifiers in FsVolumeSpi and FsDatasetSpi | Trivial | . | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HADOOP-12194](https://issues.apache.org/jira/browse/HADOOP-12194) | Support for incremental generation in the protoc plugin | Major | . | Andrew Wang | Andrew Wang |
+| [HADOOP-12180](https://issues.apache.org/jira/browse/HADOOP-12180) | Move ResourceCalculatorPlugin from YARN to Common | Major | util | Chris Douglas | Chris Douglas |
+| [HADOOP-12210](https://issues.apache.org/jira/browse/HADOOP-12210) | Collect network usage on the node | Major | . | Robert Grandl | Robert Grandl |
+| [YARN-3069](https://issues.apache.org/jira/browse/YARN-3069) | Document missing properties in yarn-default.xml | Major | documentation | Ray Chiang | Ray Chiang |
+| [YARN-3381](https://issues.apache.org/jira/browse/YARN-3381) | Fix typo InvalidStateTransitonException | Minor | api | Xiaoshuang LU | Brahma Reddy Battula |
+| [HADOOP-12211](https://issues.apache.org/jira/browse/HADOOP-12211) | Collect disks usages on the node | Major | . | Robert Grandl | Robert Grandl |
+| [HDFS-8722](https://issues.apache.org/jira/browse/HDFS-8722) | Optimize datanode writes for small writes and flushes | Critical | . | Kihwal Lee | Kihwal Lee |
+| [HADOOP-12232](https://issues.apache.org/jira/browse/HADOOP-12232) | Upgrade Tomcat dependency to 6.0.44. | Major | build | Chris Nauroth | Chris Nauroth |
+| [YARN-3170](https://issues.apache.org/jira/browse/YARN-3170) | YARN architecture document needs updating | Major | documentation | Allen Wittenauer | Brahma Reddy Battula |
+| [YARN-3174](https://issues.apache.org/jira/browse/YARN-3174) | Consolidate the NodeManager and NodeManagerRestart documentation into one | Major | documentation | Allen Wittenauer | Masatake Iwasaki |
+| [HDFS-7314](https://issues.apache.org/jira/browse/HDFS-7314) | When the DFSClient lease cannot be renewed, abort open-for-write files rather than the entire DFSClient | Major | . | Ming Ma | Ming Ma |
+| [HADOOP-10979](https://issues.apache.org/jira/browse/HADOOP-10979) | Auto-entries in hadoop\_usage | Minor | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11893](https://issues.apache.org/jira/browse/HADOOP-11893) | Mark org.apache.hadoop.security.token.Token as @InterfaceAudience.Public | Major | security | Steve Loughran | Brahma Reddy Battula |
+| [HADOOP-12081](https://issues.apache.org/jira/browse/HADOOP-12081) | Fix UserGroupInformation.java to support 64-bit zLinux | Major | security | Adam Roberts | Akira Ajisaka |
+| [HADOOP-12214](https://issues.apache.org/jira/browse/HADOOP-12214) | Parse 'HadoopArchive' commandline using cli Options. | Minor | . | Vinayakumar B | Vinayakumar B |
+| [YARN-3261](https://issues.apache.org/jira/browse/YARN-3261) | rewrite resourcemanager restart doc to remove roadmap bits | Major | documentation | Allen Wittenauer | Gururaj Shetty |
+| [YARN-2921](https://issues.apache.org/jira/browse/YARN-2921) | Fix MockRM/MockAM#waitForState sleep too long | Major | test | Karthik Kambatla | Tsuyoshi Ozawa |
+| [HADOOP-12161](https://issues.apache.org/jira/browse/HADOOP-12161) | Add getStoragePolicy API to the FileSystem interface | Major | . | Arpit Agarwal | Brahma Reddy Battula |
+| [HADOOP-12189](https://issues.apache.org/jira/browse/HADOOP-12189) | Improve CallQueueManager#swapQueue to make queue elements drop nearly impossible. | Major | ipc, test | zhihai xu | zhihai xu |
+| [HADOOP-12009](https://issues.apache.org/jira/browse/HADOOP-12009) | Clarify FileSystem.listStatus() sorting order & fix FileSystemContractBaseTest:testListStatus | Minor | documentation, fs, test | Jakob Homan | J.Andreina |
+| [HADOOP-12259](https://issues.apache.org/jira/browse/HADOOP-12259) | Utility to Dynamic port allocation | Major | test, util | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-8735](https://issues.apache.org/jira/browse/HDFS-8735) | Inotify : All events classes should implement toString() API. | Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-7858](https://issues.apache.org/jira/browse/HDFS-7858) | Improve HA Namenode Failover detection on the client | Major | hdfs-client | Arun Suresh | Arun Suresh |
+| [HDFS-8180](https://issues.apache.org/jira/browse/HDFS-8180) | AbstractFileSystem Implementation for WebHdfs | Major | webhdfs | Santhosh G Nayak | Santhosh G Nayak |
+| [HDFS-8811](https://issues.apache.org/jira/browse/HDFS-8811) | Move BlockStoragePolicy name's constants from HdfsServerConstants.java to HdfsConstants.java | Major | . | Vinayakumar B | Vinayakumar B |
+| [HDFS-8822](https://issues.apache.org/jira/browse/HDFS-8822) | Add SSD storagepolicy tests in TestBlockStoragePolicy#testDefaultPolicies | Major | . | Vinayakumar B | Vinayakumar B |
+| [YARN-3950](https://issues.apache.org/jira/browse/YARN-3950) | Add unique YARN\_SHELL\_ID environment variable to DistributedShell | Major | applications/distributed-shell | Robert Kanter | Robert Kanter |
+| [YARN-2768](https://issues.apache.org/jira/browse/YARN-2768) | Avoid cloning Resource in FSAppAttempt#updateDemand | Minor | fairscheduler | Hong Zhiguo | Hong Zhiguo |
+| [HDFS-8816](https://issues.apache.org/jira/browse/HDFS-8816) | Improve visualization for the Datanode tab in the NN UI | Major | . | Haohui Mai | Haohui Mai |
+| [HDFS-8821](https://issues.apache.org/jira/browse/HDFS-8821) | Explain message "Operation category X is not supported in state standby" | Minor | . | Gautam Gopalakrishnan | Gautam Gopalakrishnan |
+| [HADOOP-12271](https://issues.apache.org/jira/browse/HADOOP-12271) | Hadoop Jar Error Should Be More Explanatory | Minor | . | Jesse Anderson | Josh Elser |
+| [HADOOP-12249](https://issues.apache.org/jira/browse/HADOOP-12249) | pull argument parsing into a function | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12183](https://issues.apache.org/jira/browse/HADOOP-12183) | Annotate the HTrace span created by FsShell with the command-line arguments passed by the user | Minor | tracing | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-3978](https://issues.apache.org/jira/browse/YARN-3978) | Configurably turn off the saving of container info in Generic AHS | Major | timelineserver, yarn | Eric Payne | Eric Payne |
+| [YARN-3965](https://issues.apache.org/jira/browse/YARN-3965) | Ad
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/22] hadoop git commit: HDFS-10835. Fix typos in httpfs.sh.
Contributed by John Zhuge.
Posted by sh...@apache.org.
HDFS-10835. Fix typos in httpfs.sh. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c213c34
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c213c34
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c213c34
Branch: refs/heads/HADOOP-12756
Commit: 7c213c341d047362e6f8b66ab3ffafaf639d8d31
Parents: f0d5382
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Sep 6 11:44:26 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Sep 6 11:44:26 2016 -0700
----------------------------------------------------------------------
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c213c34/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
index a6c9f03..64459bf 100755
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
@@ -17,15 +17,15 @@ MYNAME="${BASH_SOURCE-$0}"
function hadoop_usage
{
- hadoop_add_subcommand "run" "Start kms in the current window"
+ hadoop_add_subcommand "run" "Start HttpFS in the current window"
hadoop_add_subcommand "run -security" "Start in the current window with security manager"
- hadoop_add_subcommand "start" "Start kms in a separate window"
+ hadoop_add_subcommand "start" "Start HttpFS in a separate window"
hadoop_add_subcommand "start -security" "Start in a separate window with security manager"
hadoop_add_subcommand "status" "Return the LSB compliant status"
- hadoop_add_subcommand "stop" "Stop kms, waiting up to 5 seconds for the process to end"
- hadoop_add_subcommand "top n" "Stop kms, waiting up to n seconds for the process to end"
- hadoop_add_subcommand "stop -force" "Stop kms, wait up to 5 seconds and then use kill -KILL if still running"
- hadoop_add_subcommand "stop n -force" "Stop kms, wait up to n seconds and then use kill -KILL if still running"
+ hadoop_add_subcommand "stop" "Stop HttpFS, waiting up to 5 seconds for the process to end"
+ hadoop_add_subcommand "stop n" "Stop HttpFS, waiting up to n seconds for the process to end"
+ hadoop_add_subcommand "stop -force" "Stop HttpFS, wait up to 5 seconds and then use kill -KILL if still running"
+ hadoop_add_subcommand "stop n -force" "Stop HttpFS, wait up to n seconds and then use kill -KILL if still running"
hadoop_generate_usage "${MYNAME}" false
}
@@ -47,7 +47,7 @@ else
exit 1
fi
-# The Java System property 'httpfs.http.port' it is not used by Kms,
+# The Java System property 'httpfs.http.port' it is not used by HttpFS,
# it is used in Tomcat's server.xml configuration file
#
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/22] hadoop git commit: YARN-5576. Allow resource localization
while container is running. Contributed by Jian He.
Posted by sh...@apache.org.
YARN-5576. Allow resource localization while container is running. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6fcfe28
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6fcfe28
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6fcfe28
Branch: refs/heads/HADOOP-12756
Commit: e6fcfe28e304062c7d09231db757acb2953703ce
Parents: 62a9667
Author: Varun Vasudev <vv...@apache.org>
Authored: Tue Sep 6 20:01:45 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Tue Sep 6 20:01:45 2016 +0530
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 9 +
.../hadoop/yarn/server/nodemanager/Context.java | 2 +
.../nodemanager/DefaultContainerExecutor.java | 6 +
.../nodemanager/DockerContainerExecutor.java | 6 +
.../nodemanager/LinuxContainerExecutor.java | 5 +
.../yarn/server/nodemanager/NodeManager.java | 12 ++
.../containermanager/ContainerManagerImpl.java | 70 +++++--
.../containermanager/container/Container.java | 8 +-
.../container/ContainerImpl.java | 207 ++++++++-----------
.../launcher/ContainerLaunch.java | 4 +-
.../localizer/ResourceLocalizationService.java | 21 +-
.../containermanager/localizer/ResourceSet.java | 192 +++++++++++++++++
.../ContainerLocalizationRequestEvent.java | 2 +-
.../localizer/event/LocalizationEventType.java | 2 +-
.../timelineservice/NMTimelinePublisher.java | 2 +-
.../nodemanager/DummyContainerManager.java | 2 +-
.../TestContainerManagerWithLCE.java | 13 +-
.../amrmproxy/BaseAMRMProxyTest.java | 6 +
.../containermanager/TestContainerManager.java | 137 +++++++++++-
.../TestResourceLocalizationService.java | 7 +-
.../TestContainersMonitorResourceChange.java | 7 +
.../nodemanager/webapp/MockContainer.java | 17 +-
22 files changed, 577 insertions(+), 160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 818b0ea..918c30a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -182,6 +182,15 @@ public abstract class ContainerExecutor implements Configurable {
throws IOException, InterruptedException;
/**
+ * Create a symlink file which points to the target.
+ * @param target The target for symlink
+ * @param symlink the symlink file
+ * @throws IOException Error when creating symlinks
+ */
+ public abstract void symLink(String target, String symlink)
+ throws IOException;
+
+ /**
* Check if a container is alive.
* @param ctx Encapsulates information necessary for container liveness check.
* @return true if container is still alive
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index 88bc29c..e888393 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -126,4 +126,6 @@ public interface Context {
void setNMTimelinePublisher(NMTimelinePublisher nmMetricsPublisher);
NMTimelinePublisher getNMTimelinePublisher();
+
+ ContainerExecutor getContainerExecutor();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 13ad9ac..9a0549d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -38,6 +38,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -511,6 +512,11 @@ public class DefaultContainerExecutor extends ContainerExecutor {
}
}
+ @Override
+ public void symLink(String target, String symlink) throws IOException {
+ FileUtil.symLink(target, symlink);
+ }
+
/** Permissions for user dir.
* $local.dir/usercache/$user */
static final short USER_PERM = (short)0750;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index ebf9566..1390214 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -490,6 +490,12 @@ public class DockerContainerExecutor extends ContainerExecutor {
}
}
+ @Override
+ public void symLink(String target, String symlink)
+ throws IOException {
+
+ }
+
/**
* Converts a directory list to a docker mount string
* @param dirs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 6890b25..cc12b20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -683,6 +683,11 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
+ public void symLink(String target, String symlink) {
+
+ }
+
+ @Override
public boolean isContainerAlive(ContainerLivenessContext ctx)
throws IOException {
String user = ctx.getUser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 280a086..7f13334 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -343,6 +343,9 @@ public class NodeManager extends CompositeService
this.context = createNMContext(containerTokenSecretManager,
nmTokenSecretManager, nmStore, isDistSchedulingEnabled, conf);
+
+ ((NMContext)context).setContainerExecutor(exec);
+
nodeLabelsProvider = createNodeLabelsProvider(conf);
if (null == nodeLabelsProvider) {
@@ -509,6 +512,7 @@ public class NodeManager extends CompositeService
private OpportunisticContainerAllocator containerAllocator;
private final QueuingContext queuingContext;
+ private ContainerExecutor executor;
private NMTimelinePublisher nmTimelinePublisher;
@@ -702,6 +706,14 @@ public class NodeManager extends CompositeService
public NMTimelinePublisher getNMTimelinePublisher() {
return nmTimelinePublisher;
}
+
+ public ContainerExecutor getContainerExecutor() {
+ return this.executor;
+ }
+
+ public void setContainerExecutor(ContainerExecutor executor) {
+ this.executor = executor;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 8c060bc..52d8566 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -18,25 +18,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager;
-import static org.apache.hadoop.service.Service.STATE.STARTED;
-
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URISyntaxException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.ByteString;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -76,6 +59,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -129,7 +113,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.SignalContainersLauncherEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEventType;
@@ -154,8 +140,25 @@ import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.ByteString;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import static org.apache.hadoop.service.Service.STATE.STARTED;
public class ContainerManagerImpl extends CompositeService implements
ContainerManager {
@@ -1525,6 +1528,31 @@ public class ContainerManagerImpl extends CompositeService implements
public ResourceLocalizationResponse localize(
ResourceLocalizationRequest request) throws YarnException, IOException {
+ ContainerId containerId = request.getContainerId();
+ Container container = context.getContainers().get(containerId);
+ if (container == null) {
+ throw new YarnException("Specified " + containerId + " does not exist!");
+ }
+ if (!container.getContainerState()
+ .equals(org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState.RUNNING)) {
+ throw new YarnException(
+ containerId + " is at " + container.getContainerState()
+ + " state. Not able to localize new resources.");
+ }
+
+ try {
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
+ container.getResourceSet().addResources(request.getLocalResources());
+ if (req != null && !req.isEmpty()) {
+ dispatcher.getEventHandler()
+ .handle(new ContainerLocalizationRequestEvent(container, req));
+ }
+ } catch (URISyntaxException e) {
+ LOG.info("Error when parsing local resource URI for " + containerId, e);
+ throw new YarnException(e);
+ }
+
return ResourceLocalizationResponse.newInstance();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index 84d3cb2..c4cea18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
-import java.util.List;
-import java.util.Map;
-
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -31,6 +28,10 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceSet;
+
+import java.util.List;
+import java.util.Map;
public interface Container extends EventHandler<ContainerEvent> {
@@ -74,4 +75,5 @@ public interface Container extends EventHandler<ContainerEvent> {
Priority getPriority();
+ ResourceSet getResourceSet();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 0244d90..ce9e581 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -18,18 +18,15 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -65,6 +62,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceSet;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationCleanupEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent;
@@ -124,20 +122,7 @@ public class ContainerImpl implements Container {
private final Configuration daemonConf;
private static final Log LOG = LogFactory.getLog(ContainerImpl.class);
- private final Map<LocalResourceRequest,List<String>> pendingResources =
- new HashMap<LocalResourceRequest,List<String>>();
- private final Map<Path,List<String>> localizedResources =
- new HashMap<Path,List<String>>();
- private final List<LocalResourceRequest> publicRsrcs =
- new ArrayList<LocalResourceRequest>();
- private final List<LocalResourceRequest> privateRsrcs =
- new ArrayList<LocalResourceRequest>();
- private final List<LocalResourceRequest> appRsrcs =
- new ArrayList<LocalResourceRequest>();
- private final Map<LocalResourceRequest, Path> resourcesToBeUploaded =
- new ConcurrentHashMap<LocalResourceRequest, Path>();
- private final Map<LocalResourceRequest, Boolean> resourcesUploadPolicies =
- new ConcurrentHashMap<LocalResourceRequest, Boolean>();
+
// whether container has been recovered after a restart
private RecoveredContainerStatus recoveredStatus =
@@ -145,6 +130,7 @@ public class ContainerImpl implements Container {
// whether container was marked as killed after recovery
private boolean recoveredAsKilled = false;
private Context context;
+ private ResourceSet resourceSet;
public ContainerImpl(Configuration conf, Dispatcher dispatcher,
ContainerLaunchContext launchContext, Credentials creds,
@@ -204,6 +190,7 @@ public class ContainerImpl implements Container {
stateMachine = stateMachineFactory.make(this);
this.context = context;
+ this.resourceSet = new ResourceSet();
}
// constructor for a recovered container
@@ -313,6 +300,12 @@ public class ContainerImpl implements Container {
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
new RetryFailureTransition())
.addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ new ResourceLocalizedWhileRunningTransition())
+ .addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
+ ContainerEventType.RESOURCE_FAILED,
+ new ResourceLocalizationFailedWhileRunningTransition())
+ .addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.RUNNING, ContainerState.KILLING,
@@ -470,7 +463,7 @@ public class ContainerImpl implements Container {
try {
if (ContainerState.LOCALIZED == getContainerState()
|| ContainerState.RELAUNCHING == getContainerState()) {
- return localizedResources;
+ return resourceSet.getLocalizedResources();
} else {
return null;
}
@@ -591,6 +584,11 @@ public class ContainerImpl implements Container {
this.logDir = logDir;
}
+ @Override
+ public ResourceSet getResourceSet() {
+ return this.resourceSet;
+ }
+
@SuppressWarnings("unchecked")
private void sendFinishedEvents() {
// Inform the application
@@ -653,7 +651,7 @@ public class ContainerImpl implements Container {
for (String s : diags) {
this.diagnostics.append(s);
}
- if (isRetryContextSet() && diagnostics.length() > diagnosticsMaxSize) {
+ if (diagnostics.length() > diagnosticsMaxSize) {
diagnostics.delete(0, diagnostics.length() - diagnosticsMaxSize);
}
try {
@@ -667,17 +665,7 @@ public class ContainerImpl implements Container {
@SuppressWarnings("unchecked") // dispatcher not typed
public void cleanup() {
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrc =
- new HashMap<LocalResourceVisibility,
- Collection<LocalResourceRequest>>();
- if (!publicRsrcs.isEmpty()) {
- rsrc.put(LocalResourceVisibility.PUBLIC, publicRsrcs);
- }
- if (!privateRsrcs.isEmpty()) {
- rsrc.put(LocalResourceVisibility.PRIVATE, privateRsrcs);
- }
- if (!appRsrcs.isEmpty()) {
- rsrc.put(LocalResourceVisibility.APPLICATION, appRsrcs);
- }
+ resourceSet.getAllResourcesByVisibility();
dispatcher.getEventHandler().handle(
new ContainerLocalizationCleanupEvent(this, rsrc));
}
@@ -697,7 +685,7 @@ public class ContainerImpl implements Container {
* message.
*
* If there are resources to localize, sends a
- * ContainerLocalizationRequest (INIT_CONTAINER_RESOURCES)
+ * ContainerLocalizationRequest (LOCALIZE_CONTAINER_RESOURCES)
* to the ResourceLocalizationManager and enters LOCALIZING state.
*
* If there are no resources to localize, sends LAUNCH_CONTAINER event
@@ -749,39 +737,15 @@ public class ContainerImpl implements Container {
}
container.containerLocalizationStartTime = clock.getTime();
+
// Send requests for public, private resources
Map<String,LocalResource> cntrRsrc = ctxt.getLocalResources();
if (!cntrRsrc.isEmpty()) {
try {
- for (Map.Entry<String,LocalResource> rsrc : cntrRsrc.entrySet()) {
- try {
- LocalResourceRequest req =
- new LocalResourceRequest(rsrc.getValue());
- List<String> links = container.pendingResources.get(req);
- if (links == null) {
- links = new ArrayList<String>();
- container.pendingResources.put(req, links);
- }
- links.add(rsrc.getKey());
- storeSharedCacheUploadPolicy(container, req, rsrc.getValue()
- .getShouldBeUploadedToSharedCache());
- switch (rsrc.getValue().getVisibility()) {
- case PUBLIC:
- container.publicRsrcs.add(req);
- break;
- case PRIVATE:
- container.privateRsrcs.add(req);
- break;
- case APPLICATION:
- container.appRsrcs.add(req);
- break;
- }
- } catch (URISyntaxException e) {
- LOG.info("Got exception parsing " + rsrc.getKey()
- + " and value " + rsrc.getValue());
- throw e;
- }
- }
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
+ container.resourceSet.addResources(ctxt.getLocalResources());
+ container.dispatcher.getEventHandler().handle(
+ new ContainerLocalizationRequestEvent(container, req));
} catch (URISyntaxException e) {
// malformed resource; abort container launch
LOG.warn("Failed to parse resource-request", e);
@@ -789,21 +753,6 @@ public class ContainerImpl implements Container {
container.metrics.endInitingContainer();
return ContainerState.LOCALIZATION_FAILED;
}
- Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
- new LinkedHashMap<LocalResourceVisibility,
- Collection<LocalResourceRequest>>();
- if (!container.publicRsrcs.isEmpty()) {
- req.put(LocalResourceVisibility.PUBLIC, container.publicRsrcs);
- }
- if (!container.privateRsrcs.isEmpty()) {
- req.put(LocalResourceVisibility.PRIVATE, container.privateRsrcs);
- }
- if (!container.appRsrcs.isEmpty()) {
- req.put(LocalResourceVisibility.APPLICATION, container.appRsrcs);
- }
-
- container.dispatcher.getEventHandler().handle(
- new ContainerLocalizationRequestEvent(container, req));
return ContainerState.LOCALIZING;
} else {
container.sendLaunchEvent();
@@ -814,27 +763,6 @@ public class ContainerImpl implements Container {
}
/**
- * Store the resource's shared cache upload policies
- * Given LocalResourceRequest can be shared across containers in
- * LocalResourcesTrackerImpl, we preserve the upload policies here.
- * In addition, it is possible for the application to create several
- * "identical" LocalResources as part of
- * ContainerLaunchContext.setLocalResources with different symlinks.
- * There is a corner case where these "identical" local resources have
- * different upload policies. For that scenario, upload policy will be set to
- * true as long as there is at least one LocalResource entry with
- * upload policy set to true.
- */
- private static void storeSharedCacheUploadPolicy(ContainerImpl container,
- LocalResourceRequest resourceRequest, Boolean uploadPolicy) {
- Boolean storedUploadPolicy =
- container.resourcesUploadPolicies.get(resourceRequest);
- if (storedUploadPolicy == null || (!storedUploadPolicy && uploadPolicy)) {
- container.resourcesUploadPolicies.put(resourceRequest, uploadPolicy);
- }
- }
-
- /**
* Transition when one of the requested resources for this container
* has been successfully localized.
*/
@@ -847,22 +775,21 @@ public class ContainerImpl implements Container {
ContainerResourceLocalizedEvent rsrcEvent = (ContainerResourceLocalizedEvent) event;
LocalResourceRequest resourceRequest = rsrcEvent.getResource();
Path location = rsrcEvent.getLocation();
- List<String> syms = container.pendingResources.remove(resourceRequest);
+ List<String> syms =
+ container.resourceSet.resourceLocalized(resourceRequest, location);
if (null == syms) {
- LOG.warn("Localized unknown resource " + resourceRequest +
- " for container " + container.containerId);
- assert false;
- // fail container?
+ LOG.info("Localized resource " + resourceRequest +
+ " for container " + container.containerId);
return ContainerState.LOCALIZING;
}
- container.localizedResources.put(location, syms);
// check to see if this resource should be uploaded to the shared cache
// as well
if (shouldBeUploadedToSharedCache(container, resourceRequest)) {
- container.resourcesToBeUploaded.put(resourceRequest, location);
+ container.resourceSet.getResourcesToBeUploaded()
+ .put(resourceRequest, location);
}
- if (!container.pendingResources.isEmpty()) {
+ if (!container.resourceSet.getPendingResources().isEmpty()) {
return ContainerState.LOCALIZING;
}
@@ -884,7 +811,8 @@ public class ContainerImpl implements Container {
&& container.recoveredStatus != RecoveredContainerStatus.COMPLETED) {
// kick off uploads to the shared cache
container.dispatcher.getEventHandler().handle(
- new SharedCacheUploadEvent(container.resourcesToBeUploaded, container
+ new SharedCacheUploadEvent(
+ container.resourceSet.getResourcesToBeUploaded(), container
.getLaunchContext(), container.getUser(),
SharedCacheUploadEventType.UPLOAD));
}
@@ -894,6 +822,56 @@ public class ContainerImpl implements Container {
}
/**
+ * Resource is localized while the container is running - create symlinks
+ */
+ static class ResourceLocalizedWhileRunningTransition
+ extends ContainerTransition {
+
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ ContainerResourceLocalizedEvent rsrcEvent =
+ (ContainerResourceLocalizedEvent) event;
+ List<String> links = container.resourceSet
+ .resourceLocalized(rsrcEvent.getResource(), rsrcEvent.getLocation());
+ // creating symlinks.
+ for (String link : links) {
+ try {
+ String linkFile = new Path(container.workDir, link).toString();
+ if (new File(linkFile).exists()) {
+ LOG.info("Symlink file already exists: " + linkFile);
+ } else {
+ container.context.getContainerExecutor()
+ .symLink(rsrcEvent.getLocation().toString(), linkFile);
+ LOG.info("Created symlink: " + linkFile + " -> " + rsrcEvent
+ .getLocation());
+ }
+ } catch (IOException e) {
+ String message = String
+ .format("Error when creating symlink %s -> %s", link,
+ rsrcEvent.getLocation());
+ LOG.error(message, e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Resource localization failed while the container is running.
+ */
+ static class ResourceLocalizationFailedWhileRunningTransition
+ extends ContainerTransition {
+
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ ContainerResourceFailedEvent failedEvent =
+ (ContainerResourceFailedEvent) event;
+ container.resourceSet
+ .resourceLocalizationFailed(failedEvent.getResource());
+ container.addDiagnostics(failedEvent.getDiagnosticMessage());
+ }
+ }
+
+ /**
* Transition from LOCALIZED state to RUNNING state upon receiving
* a CONTAINER_LAUNCHED event
*/
@@ -1136,17 +1114,10 @@ public class ContainerImpl implements Container {
SingleArcTransition<ContainerImpl, ContainerEvent> {
@Override
public void transition(ContainerImpl container, ContainerEvent event) {
- ContainerResourceLocalizedEvent rsrcEvent = (ContainerResourceLocalizedEvent) event;
- List<String> syms =
- container.pendingResources.remove(rsrcEvent.getResource());
- if (null == syms) {
- LOG.warn("Localized unknown resource " + rsrcEvent.getResource() +
- " for container " + container.containerId);
- assert false;
- // fail container?
- return;
- }
- container.localizedResources.put(rsrcEvent.getLocation(), syms);
+ ContainerResourceLocalizedEvent rsrcEvent =
+ (ContainerResourceLocalizedEvent) event;
+ container.resourceSet
+ .resourceLocalized(rsrcEvent.getResource(), rsrcEvent.getLocation());
}
}
@@ -1402,7 +1373,7 @@ public class ContainerImpl implements Container {
*/
private static boolean shouldBeUploadedToSharedCache(ContainerImpl container,
LocalResourceRequest resource) {
- return container.resourcesUploadPolicies.get(resource);
+ return container.resourceSet.getResourcesUploadPolicies().get(resource);
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 14190fc..d8239ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1201,16 +1201,16 @@ public class ContainerLaunch implements Callable<Integer> {
private void recordContainerLogDir(ContainerId containerId,
String logDir) throws IOException{
+ container.setLogDir(logDir);
if (container.isRetryContextSet()) {
- container.setLogDir(logDir);
context.getNMStateStore().storeContainerLogDir(containerId, logDir);
}
}
private void recordContainerWorkDir(ContainerId containerId,
String workDir) throws IOException{
+ container.setWorkDir(workDir);
if (container.isRetryContextSet()) {
- container.setWorkDir(workDir);
context.getNMStateStore().storeContainerWorkDir(containerId, workDir);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 0ec2f29..b281ef5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -112,6 +112,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationCleanupEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationEvent;
@@ -135,7 +136,6 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.Re
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredUserResources;
import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerBuilderUtils;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.FSDownload;
import com.google.common.annotations.VisibleForTesting;
@@ -419,7 +419,7 @@ public class ResourceLocalizationService extends CompositeService
handleInitApplicationResources(
((ApplicationLocalizationEvent)event).getApplication());
break;
- case INIT_CONTAINER_RESOURCES:
+ case LOCALIZE_CONTAINER_RESOURCES:
handleInitContainerResources((ContainerLocalizationRequestEvent) event);
break;
case CONTAINER_RESOURCES_LOCALIZED:
@@ -469,6 +469,13 @@ public class ResourceLocalizationService extends CompositeService
private void handleInitContainerResources(
ContainerLocalizationRequestEvent rsrcReqs) {
Container c = rsrcReqs.getContainer();
+ EnumSet<ContainerState> set =
+ EnumSet.of(ContainerState.LOCALIZING, ContainerState.RUNNING);
+ if (!set.contains(c.getContainerState())) {
+ LOG.warn(c.getContainerId() + " is at " + c.getContainerState()
+ + " state, do not localize resources.");
+ return;
+ }
// create a loading cache for the file statuses
LoadingCache<Path,Future<FileStatus>> statCache =
CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
@@ -538,7 +545,7 @@ public class ResourceLocalizationService extends CompositeService
}
String locId = c.getContainerId().toString();
localizerTracker.cleanupPrivLocalizers(locId);
-
+
// Delete the container directories
String userName = c.getUser();
String containerIDStr = c.toString();
@@ -747,6 +754,14 @@ public class ResourceLocalizationService extends CompositeService
case APPLICATION:
synchronized (privLocalizers) {
LocalizerRunner localizer = privLocalizers.get(locId);
+ if (localizer != null && localizer.killContainerLocalizer.get()) {
+ // Old localizer thread has been stopped, remove it and creates
+ // a new localizer thread.
+ LOG.info("New " + event.getType() + " localize request for "
+ + locId + ", remove old private localizer.");
+ cleanupPrivLocalizers(locId);
+ localizer = null;
+ }
if (null == localizer) {
LOG.info("Created localizer for " + locId);
localizer = new LocalizerRunner(req.getContext(), locId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
new file mode 100644
index 0000000..a41ee20
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * All Resources requested by the container.
+ */
+public class ResourceSet {
+
+ private static final Log LOG = LogFactory.getLog(ResourceSet.class);
+
+ // resources by localization state (localized, pending, failed)
+ private Map<Path, List<String>> localizedResources =
+ new ConcurrentHashMap<>();
+ private Map<LocalResourceRequest, List<String>> pendingResources =
+ new ConcurrentHashMap<>();
+ private Set<LocalResourceRequest> resourcesFailedToBeLocalized =
+ new HashSet<>();
+
+ // resources by visibility (public, private, app)
+ private final List<LocalResourceRequest> publicRsrcs =
+ new ArrayList<>();
+ private final List<LocalResourceRequest> privateRsrcs =
+ new ArrayList<>();
+ private final List<LocalResourceRequest> appRsrcs =
+ new ArrayList<>();
+
+ private final Map<LocalResourceRequest, Path> resourcesToBeUploaded =
+ new ConcurrentHashMap<>();
+ private final Map<LocalResourceRequest, Boolean> resourcesUploadPolicies =
+ new ConcurrentHashMap<>();
+
+ public Map<LocalResourceVisibility, Collection<LocalResourceRequest>>
+ addResources(Map<String, LocalResource> localResourceMap)
+ throws URISyntaxException {
+ if (localResourceMap == null || localResourceMap.isEmpty()) {
+ return null;
+ }
+ Map<LocalResourceRequest, List<String>> allResources = new HashMap<>();
+ List<LocalResourceRequest> publicList = new ArrayList<>();
+ List<LocalResourceRequest> privateList = new ArrayList<>();
+ List<LocalResourceRequest> appList = new ArrayList<>();
+
+ for (Map.Entry<String, LocalResource> rsrc : localResourceMap.entrySet()) {
+ LocalResource resource = rsrc.getValue();
+ LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue());
+ allResources.putIfAbsent(req, new ArrayList<>());
+ allResources.get(req).add(rsrc.getKey());
+ storeSharedCacheUploadPolicy(req,
+ resource.getShouldBeUploadedToSharedCache());
+ switch (resource.getVisibility()) {
+ case PUBLIC:
+ publicList.add(req);
+ break;
+ case PRIVATE:
+ privateList.add(req);
+ break;
+ case APPLICATION:
+ appList.add(req);
+ break;
+ default:
+ break;
+ }
+ }
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
+ new LinkedHashMap<>();
+ if (!publicList.isEmpty()) {
+ publicRsrcs.addAll(publicList);
+ req.put(LocalResourceVisibility.PUBLIC, publicList);
+ }
+ if (!privateList.isEmpty()) {
+ privateRsrcs.addAll(privateList);
+ req.put(LocalResourceVisibility.PRIVATE, privateList);
+ }
+ if (!appList.isEmpty()) {
+ appRsrcs.addAll(appList);
+ req.put(LocalResourceVisibility.APPLICATION, appList);
+ }
+ if (!allResources.isEmpty()) {
+ this.pendingResources.putAll(allResources);
+ }
+ return req;
+ }
+
+ /**
+ * Called when resource localized.
+ * @param request The original request for the localized resource
+ * @param location The path where the resource is localized
+ * @return The list of symlinks for the localized resources.
+ */
+ public List<String> resourceLocalized(LocalResourceRequest request,
+ Path location) {
+ List<String> symlinks = pendingResources.remove(request);
+ if (symlinks == null) {
+ return null;
+ } else {
+ localizedResources.put(location, symlinks);
+ return symlinks;
+ }
+ }
+
+ public void resourceLocalizationFailed(LocalResourceRequest request) {
+ pendingResources.remove(request);
+ resourcesFailedToBeLocalized.add(request);
+ }
+
+ public synchronized Map<LocalResourceVisibility,
+ Collection<LocalResourceRequest>> getAllResourcesByVisibility() {
+
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrc =
+ new HashMap<>();
+ if (!publicRsrcs.isEmpty()) {
+ rsrc.put(LocalResourceVisibility.PUBLIC, publicRsrcs);
+ }
+ if (!privateRsrcs.isEmpty()) {
+ rsrc.put(LocalResourceVisibility.PRIVATE, privateRsrcs);
+ }
+ if (!appRsrcs.isEmpty()) {
+ rsrc.put(LocalResourceVisibility.APPLICATION, appRsrcs);
+ }
+ return rsrc;
+ }
+
+ /**
+ * Store the resource's shared cache upload policies
+ * Given LocalResourceRequest can be shared across containers in
+ * LocalResourcesTrackerImpl, we preserve the upload policies here.
+ * In addition, it is possible for the application to create several
+ * "identical" LocalResources as part of
+ * ContainerLaunchContext.setLocalResources with different symlinks.
+ * There is a corner case where these "identical" local resources have
+ * different upload policies. For that scenario, upload policy will be set to
+ * true as long as there is at least one LocalResource entry with
+ * upload policy set to true.
+ */
+ private void storeSharedCacheUploadPolicy(
+ LocalResourceRequest resourceRequest, Boolean uploadPolicy) {
+ Boolean storedUploadPolicy = resourcesUploadPolicies.get(resourceRequest);
+ if (storedUploadPolicy == null || (!storedUploadPolicy && uploadPolicy)) {
+ resourcesUploadPolicies.put(resourceRequest, uploadPolicy);
+ }
+ }
+
+ public Map<Path, List<String>> getLocalizedResources() {
+ return localizedResources;
+ }
+
+ public Map<LocalResourceRequest, Path> getResourcesToBeUploaded() {
+ return resourcesToBeUploaded;
+ }
+
+ public Map<LocalResourceRequest, Boolean> getResourcesUploadPolicies() {
+ return resourcesUploadPolicies;
+ }
+
+ public Map<LocalResourceRequest, List<String>> getPendingResources() {
+ return pendingResources;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
index 11bb25e..43a2f33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
@@ -44,7 +44,7 @@ public class ContainerLocalizationRequestEvent extends
*/
public ContainerLocalizationRequestEvent(Container c,
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrc) {
- super(LocalizationEventType.INIT_CONTAINER_RESOURCES, c);
+ super(LocalizationEventType.LOCALIZE_CONTAINER_RESOURCES, c);
this.rsrc = rsrc;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
index 4785fba..cb9842d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.eve
public enum LocalizationEventType {
INIT_APPLICATION_RESOURCES,
- INIT_CONTAINER_RESOURCES,
+ LOCALIZE_CONTAINER_RESOURCES,
CACHE_CLEANUP,
CLEANUP_CONTAINER_RESOURCES,
DESTROY_APPLICATION_RESOURCES,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index 0371863..8e68889 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -326,7 +326,7 @@ public class NMTimelinePublisher extends CompositeService {
publishContainerLocalizationEvent((ContainerLocalizationEvent) event,
ContainerMetricsConstants.LOCALIZATION_FINISHED_EVENT_TYPE);
break;
- case INIT_CONTAINER_RESOURCES:
+ case LOCALIZE_CONTAINER_RESOURCES:
publishContainerLocalizationEvent((ContainerLocalizationEvent) event,
ContainerMetricsConstants.LOCALIZATION_START_EVENT_TYPE);
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
index 4aeb2e2..740ed19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
@@ -86,7 +86,7 @@ public class DummyContainerManager extends ContainerManagerImpl {
dispatcher.getEventHandler().handle(new ApplicationInitedEvent(
app.getAppId()));
break;
- case INIT_CONTAINER_RESOURCES:
+ case LOCALIZE_CONTAINER_RESOURCES:
ContainerLocalizationRequestEvent rsrcReqs =
(ContainerLocalizationRequestEvent) event;
// simulate localization of all requested resources
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
index 3e00885..aa0d975 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
@@ -131,7 +131,18 @@ public class TestContainerManagerWithLCE extends TestContainerManager {
LOG.info("Running testContainerLaunchAndExitFailure");
super.testContainerLaunchAndExitFailure();
}
-
+
+ @Override
+ public void testLocalingResourceWhileContainerRunning()
+ throws Exception {
+ // Don't run the test if the binary is not available.
+ if (!shouldRunTest()) {
+ LOG.info("LCE binary path is not passed. Not running the test");
+ return;
+ }
+ super.testLocalingResourceWhileContainerRunning();
+ }
+
@Override
public void testLocalFilesCleanup() throws InterruptedException,
IOException, YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index d8660dd..031300f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitor;
@@ -715,6 +716,11 @@ public abstract class BaseAMRMProxyTest {
@Override
public NMTimelinePublisher getNMTimelinePublisher() {
+ return null;
+ }
+
+ @Override
+ public ContainerExecutor getContainerExecutor() {
return null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index 1f803b4..5785e1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -36,17 +36,20 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Supplier;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
@@ -83,6 +86,7 @@ import org.apache.hadoop.yarn.server.nodemanager.CMgrSignalContainersEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestAuxServices.ServiceA;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -459,7 +463,138 @@ public class TestContainerManager extends BaseContainerManagerTest {
// and verify exit code returned
testContainerLaunchAndExit(exitCode);
}
-
+
+ private Map<String, LocalResource> setupLocalResources(String fileName,
+ String symLink) throws Exception {
+ // ////// Create the resources for the container
+ File dir = new File(tmpDir, "dir");
+ dir.mkdirs();
+ File file = new File(dir, fileName);
+ PrintWriter fileWriter = new PrintWriter(file);
+ fileWriter.write("Hello World!");
+ fileWriter.close();
+
+ URL resourceURL = URL.fromPath(FileContext.getLocalFSFileContext()
+ .makeQualified(new Path(file.getAbsolutePath())));
+ LocalResource resource =
+ recordFactory.newRecordInstance(LocalResource.class);
+ resource.setResource(resourceURL);
+ resource.setSize(-1);
+ resource.setVisibility(LocalResourceVisibility.APPLICATION);
+ resource.setType(LocalResourceType.FILE);
+ resource.setTimestamp(file.lastModified());
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ localResources.put(symLink, resource);
+ return localResources;
+ }
+
+ // Start the container
+ // While the container is running, localize new resources.
+ // Verify the symlink is created properly
+ @Test
+ public void testLocalingResourceWhileContainerRunning() throws Exception {
+ // Real del service
+ delSrvc = new DeletionService(exec);
+ delSrvc.init(conf);
+
+ ((NodeManager.NMContext)context).setContainerExecutor(exec);
+ containerManager = createContainerManager(delSrvc);
+ containerManager.init(conf);
+ containerManager.start();
+ // set up local resources
+ Map<String, LocalResource> localResource =
+ setupLocalResources("file", "symLink1");
+ ContainerLaunchContext context =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ context.setLocalResources(localResource);
+
+ // a long running container - sleep
+ context.setCommands(Arrays.asList("sleep 6"));
+ ContainerId cId = createContainerId(0);
+
+ // start the container
+ StartContainerRequest scRequest = StartContainerRequest.newInstance(context,
+ createContainerToken(cId, DUMMY_RM_IDENTIFIER, this.context.getNodeId(),
+ user, this.context.getContainerTokenSecretManager()));
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(Arrays.asList(scRequest));
+ containerManager.startContainers(allRequests);
+ BaseContainerManagerTest
+ .waitForContainerState(containerManager, cId, ContainerState.RUNNING);
+
+ BaseContainerManagerTest.waitForApplicationState(containerManager,
+ cId.getApplicationAttemptId().getApplicationId(),
+ ApplicationState.RUNNING);
+ checkResourceLocalized(cId, "symLink1");
+
+ // Localize new local resources while container is running
+ Map<String, LocalResource> localResource2 =
+ setupLocalResources("file2", "symLink2");
+
+ ResourceLocalizationRequest request =
+ ResourceLocalizationRequest.newInstance(cId, localResource2);
+ containerManager.localize(request);
+
+ // Verify resource is localized and symlink is created.
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ try {
+ checkResourceLocalized(cId, "symLink2");
+ return true;
+ } catch (Throwable e) {
+ return false;
+ }
+ }
+ }, 500, 20000);
+
+ BaseContainerManagerTest
+ .waitForContainerState(containerManager, cId, ContainerState.COMPLETE);
+ // Verify container cannot localize resources while at non-running state.
+ try{
+ containerManager.localize(request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().contains("Not able to localize new resources"));
+ }
+ }
+
+ private void checkResourceLocalized(ContainerId containerId, String symLink) {
+ String appId =
+ containerId.getApplicationAttemptId().getApplicationId().toString();
+ File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
+ File userDir = new File(userCacheDir, user);
+ File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
+ // localDir/usercache/nobody/appcache/application_0_0000
+ File appDir = new File(appCache, appId);
+ // localDir/usercache/nobody/appcache/application_0_0000/container_0_0000_01_000000
+ File containerDir = new File(appDir, containerId.toString());
+ // localDir/usercache/nobody/appcache/application_0_0000/container_0_0000_01_000000/symLink1
+ File targetFile = new File(containerDir, symLink);
+
+ File sysDir =
+ new File(localDir, ResourceLocalizationService.NM_PRIVATE_DIR);
+ // localDir/nmPrivate/application_0_0000
+ File appSysDir = new File(sysDir, appId);
+ // localDir/nmPrivate/application_0_0000/container_0_0000_01_000000
+ File containerSysDir = new File(appSysDir, containerId.toString());
+
+ Assert.assertTrue("AppDir " + appDir.getAbsolutePath() + " doesn't exist!!",
+ appDir.exists());
+ Assert.assertTrue(
+ "AppSysDir " + appSysDir.getAbsolutePath() + " doesn't exist!!",
+ appSysDir.exists());
+ Assert.assertTrue(
+ "containerDir " + containerDir.getAbsolutePath() + " doesn't exist !",
+ containerDir.exists());
+ Assert.assertTrue("containerSysDir " + containerSysDir.getAbsolutePath()
+ + " doesn't exist !", containerDir.exists());
+ Assert.assertTrue(
+ "targetFile " + targetFile.getAbsolutePath() + " doesn't exist !!",
+ targetFile.exists());
+ }
+
@Test
public void testLocalFilesCleanup() throws InterruptedException,
IOException, YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index 23786fe..7aa657e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -66,6 +66,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
import org.junit.Assert;
@@ -1767,7 +1768,7 @@ public class TestResourceLocalizationService {
// creating new containers and populating corresponding localizer runners
// Container - 1
- ContainerImpl container1 = createMockContainer(user, 1);
+ Container container1 = createMockContainer(user, 1);
String localizerId1 = container1.getContainerId().toString();
rls.getPrivateLocalizers().put(
localizerId1,
@@ -2292,7 +2293,7 @@ public class TestResourceLocalizationService {
}
private ContainerLocalizationRequestEvent createContainerLocalizationEvent(
- ContainerImpl container, LocalResourceVisibility vis,
+ Container container, LocalResourceVisibility vis,
LocalResourceRequest req) {
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> reqs =
new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
@@ -2310,6 +2311,7 @@ public class TestResourceLocalizationService {
when(container.getUser()).thenReturn(user);
Credentials mockCredentials = mock(Credentials.class);
when(container.getCredentials()).thenReturn(mockCredentials);
+ when(container.getContainerState()).thenReturn(ContainerState.LOCALIZING);
return container;
}
@@ -2358,6 +2360,7 @@ public class TestResourceLocalizationService {
creds.addToken(new Text("tok" + id), tk);
when(c.getCredentials()).thenReturn(creds);
when(c.toString()).thenReturn(cId.toString());
+ when(c.getContainerState()).thenReturn(ContainerState.LOCALIZING);
return c;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index 2df0c98..d24f89d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -89,6 +89,13 @@ public class TestContainersMonitorResourceChange {
public void deleteAsUser(DeletionAsUserContext ctx)
throws IOException, InterruptedException {
}
+
+ @Override
+ public void symLink(String target, String symlink)
+ throws IOException {
+
+ }
+
@Override
public String getProcessId(ContainerId containerId) {
return String.valueOf(containerId.getContainerId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index 8332b2a..c176556 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -18,11 +19,6 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
@@ -41,8 +37,14 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceSet;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
public class MockContainer implements Container {
private ContainerId id;
@@ -119,6 +121,11 @@ public class MockContainer implements Container {
}
@Override
+ public ResourceSet getResourceSet() {
+ return null;
+ }
+
+ @Override
public void handle(ContainerEvent event) {
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/22] hadoop git commit: HADOOP-13218. Migrate other Hadoop side
tests to prepare for removing WritableRPCEngine. Contributed by Wei Zhou and
Kai Zheng
Posted by sh...@apache.org.
HADOOP-13218. Migrate other Hadoop side tests to prepare for removing WritableRPCEngine. Contributed by Wei Zhou and Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62a96671
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62a96671
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62a96671
Branch: refs/heads/HADOOP-12756
Commit: 62a9667136ebd8a048f556b534fcff4fdaf8e2ec
Parents: f955712
Author: Kai Zheng <ka...@intel.com>
Authored: Wed Sep 7 17:05:33 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Wed Sep 7 17:05:33 2016 +0800
----------------------------------------------------------------------
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 5 +-
.../main/java/org/apache/hadoop/ipc/RPC.java | 15 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 4 +-
.../hadoop/security/UserGroupInformation.java | 4 +-
.../org/apache/hadoop/ipc/RPCCallBenchmark.java | 38 +--
.../hadoop/ipc/TestMultipleProtocolServer.java | 236 +--------------
.../apache/hadoop/ipc/TestRPCCallBenchmark.java | 13 -
.../apache/hadoop/ipc/TestRPCCompatibility.java | 242 ++-------------
.../apache/hadoop/ipc/TestRPCWaitForProxy.java | 37 ++-
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 50 +++-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 74 +++--
.../hadoop/security/TestDoAsEffectiveUser.java | 291 +++++++------------
.../security/TestUserGroupInformation.java | 28 +-
.../hadoop-common/src/test/proto/test.proto | 4 +-
.../src/test/proto/test_rpc_service.proto | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 3 -
.../TestClientProtocolWithDelegationToken.java | 119 --------
.../mapreduce/v2/hs/server/HSAdminServer.java | 3 -
18 files changed, 294 insertions(+), 876 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index 83e4b9e..e68bfd4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -60,7 +60,7 @@ public class ProtobufRpcEngine implements RpcEngine {
private static final ThreadLocal<AsyncGet<Message, Exception>>
ASYNC_RETURN_MESSAGE = new ThreadLocal<>();
- static { // Register the rpcRequest deserializer for WritableRpcEngine
+ static { // Register the rpcRequest deserializer for ProtobufRpcEngine
org.apache.hadoop.ipc.Server.registerProtocolEngine(
RPC.RpcKind.RPC_PROTOCOL_BUFFER, RpcProtobufRequest.class,
new Server.ProtoBufRpcInvoker());
@@ -194,7 +194,8 @@ public class ProtobufRpcEngine implements RpcEngine {
}
if (args.length != 2) { // RpcController + Message
- throw new ServiceException("Too many parameters for request. Method: ["
+ throw new ServiceException(
+ "Too many or few parameters for request. Method: ["
+ method.getName() + "]" + ", Expected: 2, Actual: "
+ args.length);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
index 3f68d63..12a07a5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.ipc;
+import java.io.IOException;
+import java.io.InterruptedIOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
@@ -26,7 +28,6 @@ import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.net.NoRouteToHostException;
import java.net.SocketTimeoutException;
-import java.io.*;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.Arrays;
@@ -37,11 +38,12 @@ import java.util.concurrent.atomic.AtomicBoolean;
import javax.net.SocketFactory;
-import org.apache.commons.logging.*;
-
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.io.*;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolInfoService;
@@ -54,7 +56,6 @@ import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
@@ -87,7 +88,7 @@ public class RPC {
RPC_WRITABLE ((short) 2), // Use WritableRpcEngine
RPC_PROTOCOL_BUFFER ((short) 3); // Use ProtobufRpcEngine
final static short MAX_INDEX = RPC_PROTOCOL_BUFFER.value; // used for array size
- public final short value; //TODO make it private
+ private final short value;
RpcKind(short val) {
this.value = val;
@@ -207,7 +208,7 @@ public class RPC {
RpcEngine engine = PROTOCOL_ENGINES.get(protocol);
if (engine == null) {
Class<?> impl = conf.getClass(ENGINE_PROP+"."+protocol.getName(),
- WritableRpcEngine.class);
+ ProtobufRpcEngine.class);
engine = (RpcEngine)ReflectionUtils.newInstance(impl, conf);
PROTOCOL_ENGINES.put(protocol, engine);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 80b3303..ea4f533 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -236,14 +236,14 @@ public abstract class Server {
static class RpcKindMapValue {
final Class<? extends Writable> rpcRequestWrapperClass;
final RpcInvoker rpcInvoker;
+
RpcKindMapValue (Class<? extends Writable> rpcRequestWrapperClass,
RpcInvoker rpcInvoker) {
this.rpcInvoker = rpcInvoker;
this.rpcRequestWrapperClass = rpcRequestWrapperClass;
}
}
- static Map<RPC.RpcKind, RpcKindMapValue> rpcKindMap = new
- HashMap<RPC.RpcKind, RpcKindMapValue>(4);
+ static Map<RPC.RpcKind, RpcKindMapValue> rpcKindMap = new HashMap<>(4);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 24c75be..fe6fbe4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -715,7 +715,7 @@ public class UserGroupInformation {
*
* @param user The principal name to load from the ticket
* cache
- * @param ticketCachePath the path to the ticket cache file
+ * @param ticketCache the path to the ticket cache file
*
* @throws IOException if the kerberos login fails
*/
@@ -775,7 +775,7 @@ public class UserGroupInformation {
/**
* Create a UserGroupInformation from a Subject with Kerberos principal.
*
- * @param user The KerberosPrincipal to use in UGI
+ * @param subject The KerberosPrincipal to use in UGI
*
* @throws IOException if the kerberos login fails
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
index eb7b949..9356dab 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
@@ -17,13 +17,8 @@
*/
package org.apache.hadoop.ipc;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadMXBean;
-import java.net.InetSocketAddress;
-import java.security.PrivilegedExceptionAction;
-import java.util.concurrent.atomic.AtomicLong;
-
+import com.google.common.base.Joiner;
+import com.google.protobuf.BlockingService;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -34,7 +29,6 @@ import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ipc.RPC.Server;
-import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoRequestProto;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoResponseProto;
import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
@@ -45,8 +39,12 @@ import org.apache.hadoop.test.MultithreadedTestUtil.TestContext;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import com.google.common.base.Joiner;
-import com.google.protobuf.BlockingService;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadMXBean;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.atomic.AtomicLong;
/**
* Benchmark for protobuf RPC.
@@ -68,7 +66,7 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
public int secondsToRun = 15;
private int msgSize = 1024;
public Class<? extends RpcEngine> rpcEngine =
- WritableRpcEngine.class;
+ ProtobufRpcEngine.class;
private MyOptions(String args[]) {
try {
@@ -135,7 +133,7 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
opts.addOption(
OptionBuilder.withLongOpt("engine").hasArg(true)
- .withArgName("writable|protobuf")
+ .withArgName("protobuf")
.withDescription("engine to use")
.create('e'));
@@ -184,8 +182,6 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
String eng = line.getOptionValue('e');
if ("protobuf".equals(eng)) {
rpcEngine = ProtobufRpcEngine.class;
- } else if ("writable".equals(eng)) {
- rpcEngine = WritableRpcEngine.class;
} else {
throw new ParseException("invalid engine: " + eng);
}
@@ -237,11 +233,6 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
server = new RPC.Builder(conf).setProtocol(TestRpcService.class)
.setInstance(service).setBindAddress(opts.host).setPort(opts.getPort())
.setNumHandlers(opts.serverThreads).setVerbose(false).build();
- } else if (opts.rpcEngine == WritableRpcEngine.class) {
- server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestRPC.TestImpl()).setBindAddress(opts.host)
- .setPort(opts.getPort()).setNumHandlers(opts.serverThreads)
- .setVerbose(false).build();
} else {
throw new RuntimeException("Bad engine: " + opts.rpcEngine);
}
@@ -399,15 +390,6 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
return responseProto.getMessage();
}
};
- } else if (opts.rpcEngine == WritableRpcEngine.class) {
- final TestProtocol proxy = RPC.getProxy(
- TestProtocol.class, TestProtocol.versionID, addr, conf);
- return new RpcServiceWrapper() {
- @Override
- public String doEcho(String msg) throws Exception {
- return proxy.echo(msg);
- }
- };
} else {
throw new RuntimeException("unsupported engine: " + opts.rpcEngine);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
index 8b419e3..10e23ba 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
@@ -17,252 +17,28 @@
*/
package org.apache.hadoop.ipc;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
-import org.apache.hadoop.net.NetUtils;
-import org.junit.Before;
import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
-import com.google.protobuf.BlockingService;
public class TestMultipleProtocolServer extends TestRpcBase {
- private static InetSocketAddress addr;
- private static RPC.Server server;
-
- private static Configuration conf = new Configuration();
-
-
- @ProtocolInfo(protocolName="Foo")
- interface Foo0 extends VersionedProtocol {
- public static final long versionID = 0L;
- String ping() throws IOException;
-
- }
-
- @ProtocolInfo(protocolName="Foo")
- interface Foo1 extends VersionedProtocol {
- public static final long versionID = 1L;
- String ping() throws IOException;
- String ping2() throws IOException;
- }
-
- @ProtocolInfo(protocolName="Foo")
- interface FooUnimplemented extends VersionedProtocol {
- public static final long versionID = 2L;
- String ping() throws IOException;
- }
-
- interface Mixin extends VersionedProtocol{
- public static final long versionID = 0L;
- void hello() throws IOException;
- }
-
- interface Bar extends Mixin {
- public static final long versionID = 0L;
- int echo(int i) throws IOException;
- }
-
- class Foo0Impl implements Foo0 {
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return Foo0.versionID;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- Class<? extends VersionedProtocol> inter;
- try {
- inter = (Class<? extends VersionedProtocol>)getClass().
- getGenericInterfaces()[0];
- } catch (Exception e) {
- throw new IOException(e);
- }
- return ProtocolSignature.getProtocolSignature(clientMethodsHash,
- getProtocolVersion(protocol, clientVersion), inter);
- }
-
- @Override
- public String ping() {
- return "Foo0";
- }
-
- }
-
- class Foo1Impl implements Foo1 {
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return Foo1.versionID;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- Class<? extends VersionedProtocol> inter;
- try {
- inter = (Class<? extends VersionedProtocol>)getClass().
- getGenericInterfaces()[0];
- } catch (Exception e) {
- throw new IOException(e);
- }
- return ProtocolSignature.getProtocolSignature(clientMethodsHash,
- getProtocolVersion(protocol, clientVersion), inter);
- }
-
- @Override
- public String ping() {
- return "Foo1";
- }
- @Override
- public String ping2() {
- return "Foo1";
-
- }
-
- }
-
-
- class BarImpl implements Bar {
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return Bar.versionID;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- Class<? extends VersionedProtocol> inter;
- try {
- inter = (Class<? extends VersionedProtocol>)getClass().
- getGenericInterfaces()[0];
- } catch (Exception e) {
- throw new IOException(e);
- }
- return ProtocolSignature.getProtocolSignature(clientMethodsHash,
- getProtocolVersion(protocol, clientVersion), inter);
- }
-
- @Override
- public int echo(int i) {
- return i;
- }
-
- @Override
- public void hello() {
+ private static RPC.Server server;
-
- }
- }
@Before
public void setUp() throws Exception {
- // create a server with two handlers
- server = new RPC.Builder(conf).setProtocol(Foo0.class)
- .setInstance(new Foo0Impl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Foo1.class, new Foo1Impl());
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Bar.class, new BarImpl());
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Mixin.class, new BarImpl());
-
-
- // Add Protobuf server
- // Create server side implementation
- PBServerImpl pbServerImpl = new PBServerImpl();
- BlockingService service = TestProtobufRpcProto
- .newReflectiveBlockingService(pbServerImpl);
- server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, TestRpcService.class,
- service);
- server.start();
- addr = NetUtils.getConnectAddress(server);
+ super.setupConf();
+
+ server = setupTestServer(conf, 2);
}
-
+
@After
public void tearDown() throws Exception {
server.stop();
}
- @Test
- public void test1() throws IOException {
- ProtocolProxy<?> proxy;
- proxy = RPC.getProtocolProxy(Foo0.class, Foo0.versionID, addr, conf);
- Foo0 foo0 = (Foo0)proxy.getProxy();
- Assert.assertEquals("Foo0", foo0.ping());
-
-
- proxy = RPC.getProtocolProxy(Foo1.class, Foo1.versionID, addr, conf);
-
-
- Foo1 foo1 = (Foo1)proxy.getProxy();
- Assert.assertEquals("Foo1", foo1.ping());
- Assert.assertEquals("Foo1", foo1.ping());
-
-
- proxy = RPC.getProtocolProxy(Bar.class, Foo1.versionID, addr, conf);
-
-
- Bar bar = (Bar)proxy.getProxy();
- Assert.assertEquals(99, bar.echo(99));
-
- // Now test Mixin class method
-
- Mixin mixin = bar;
- mixin.hello();
- }
-
-
- // Server does not implement the FooUnimplemented version of protocol Foo.
- // See that calls to it fail.
- @Test(expected=IOException.class)
- public void testNonExistingProtocol() throws IOException {
- ProtocolProxy<?> proxy;
- proxy = RPC.getProtocolProxy(FooUnimplemented.class,
- FooUnimplemented.versionID, addr, conf);
-
- FooUnimplemented foo = (FooUnimplemented)proxy.getProxy();
- foo.ping();
- }
-
- /**
- * getProtocolVersion of an unimplemented version should return highest version
- * Similarly getProtocolSignature should work.
- * @throws IOException
- */
- @Test
- public void testNonExistingProtocol2() throws IOException {
- ProtocolProxy<?> proxy;
- proxy = RPC.getProtocolProxy(FooUnimplemented.class,
- FooUnimplemented.versionID, addr, conf);
-
- FooUnimplemented foo = (FooUnimplemented)proxy.getProxy();
- Assert.assertEquals(Foo1.versionID,
- foo.getProtocolVersion(RPC.getProtocolName(FooUnimplemented.class),
- FooUnimplemented.versionID));
- foo.getProtocolSignature(RPC.getProtocolName(FooUnimplemented.class),
- FooUnimplemented.versionID, 0);
- }
-
- @Test(expected=IOException.class)
- public void testIncorrectServerCreation() throws IOException {
- new RPC.Builder(conf).setProtocol(Foo1.class).setInstance(new Foo0Impl())
- .setBindAddress(ADDRESS).setPort(0).setNumHandlers(2).setVerbose(false)
- .build();
- }
-
// Now test a PB service - a server hosts both PB and Writable Rpcs.
@Test
public void testPBService() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
index 969f728..6d83d7d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
@@ -26,19 +26,6 @@ import org.junit.Test;
public class TestRPCCallBenchmark {
@Test(timeout=20000)
- public void testBenchmarkWithWritable() throws Exception {
- int rc = ToolRunner.run(new RPCCallBenchmark(),
- new String[] {
- "--clientThreads", "30",
- "--serverThreads", "30",
- "--time", "5",
- "--serverReaderThreads", "4",
- "--messageSize", "1024",
- "--engine", "writable"});
- assertEquals(0, rc);
- }
-
- @Test(timeout=20000)
public void testBenchmarkWithProto() throws Exception {
int rc = ToolRunner.run(new RPCCallBenchmark(),
new String[] {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
index 2ac2be9..a06d9fd 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
@@ -18,28 +18,20 @@
package org.apache.hadoop.ipc;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-
-import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureRequestProto;
-import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureResponseProto;
-import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolSignatureProto;
-import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
-import org.apache.hadoop.net.NetUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
/** Unit test for supporting method-name based compatible RPCs. */
public class TestRPCCompatibility {
private static final String ADDRESS = "0.0.0.0";
@@ -49,7 +41,7 @@ public class TestRPCCompatibility {
public static final Log LOG =
LogFactory.getLog(TestRPCCompatibility.class);
-
+
private static Configuration conf = new Configuration();
public interface TestProtocol0 extends VersionedProtocol {
@@ -120,6 +112,21 @@ public class TestRPCCompatibility {
@Before
public void setUp() {
ProtocolSignature.resetCache();
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol0.class, ProtobufRpcEngine.class);
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol1.class, ProtobufRpcEngine.class);
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol2.class, ProtobufRpcEngine.class);
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol3.class, ProtobufRpcEngine.class);
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol4.class, ProtobufRpcEngine.class);
}
@After
@@ -133,117 +140,7 @@ public class TestRPCCompatibility {
server = null;
}
}
-
- @Test // old client vs new server
- public void testVersion0ClientVersion1Server() throws Exception {
- // create a server with two handlers
- TestImpl1 impl = new TestImpl1();
- server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
- .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
- .setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- proxy = RPC.getProtocolProxy(
- TestProtocol0.class, TestProtocol0.versionID, addr, conf);
-
- TestProtocol0 proxy0 = (TestProtocol0)proxy.getProxy();
- proxy0.ping();
- }
-
- @Test // old client vs new server
- public void testVersion1ClientVersion0Server() throws Exception {
- // create a server with two handlers
- server = new RPC.Builder(conf).setProtocol(TestProtocol0.class)
- .setInstance(new TestImpl0()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- proxy = RPC.getProtocolProxy(
- TestProtocol1.class, TestProtocol1.versionID, addr, conf);
-
- TestProtocol1 proxy1 = (TestProtocol1)proxy.getProxy();
- proxy1.ping();
- try {
- proxy1.echo("hello");
- fail("Echo should fail");
- } catch(IOException e) {
- }
- }
-
- private class Version2Client {
- private TestProtocol2 proxy2;
- private ProtocolProxy<TestProtocol2> serverInfo;
-
- private Version2Client() throws IOException {
- serverInfo = RPC.getProtocolProxy(
- TestProtocol2.class, TestProtocol2.versionID, addr, conf);
- proxy2 = serverInfo.getProxy();
- }
-
- public int echo(int value) throws IOException, NumberFormatException {
- if (serverInfo.isMethodSupported("echo", int.class)) {
-System.out.println("echo int is supported");
- return -value; // use version 3 echo long
- } else { // server is version 2
-System.out.println("echo int is NOT supported");
- return Integer.parseInt(proxy2.echo(String.valueOf(value)));
- }
- }
-
- public String echo(String value) throws IOException {
- return proxy2.echo(value);
- }
-
- public void ping() throws IOException {
- proxy2.ping();
- }
- }
-
- @Test // Compatible new client & old server
- public void testVersion2ClientVersion1Server() throws Exception {
- // create a server with two handlers
- TestImpl1 impl = new TestImpl1();
- server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
- .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
- .setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
-
- Version2Client client = new Version2Client();
- client.ping();
- assertEquals("hello", client.echo("hello"));
-
- // echo(int) is not supported by server, so returning 3
- // This verifies that echo(int) and echo(String)'s hash codes are different
- assertEquals(3, client.echo(3));
- }
-
- @Test // equal version client and server
- public void testVersion2ClientVersion2Server() throws Exception {
- // create a server with two handlers
- TestImpl2 impl = new TestImpl2();
- server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
- .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
- .setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- Version2Client client = new Version2Client();
-
- client.ping();
- assertEquals("hello", client.echo("hello"));
-
- // now that echo(int) is supported by the server, echo(int) should return -3
- assertEquals(-3, client.echo(3));
- }
-
public interface TestProtocol3 {
int echo(String value);
int echo(int value);
@@ -297,97 +194,4 @@ System.out.println("echo int is NOT supported");
@Override
int echo(int value) throws IOException;
}
-
- @Test
- public void testVersionMismatch() throws IOException {
- server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
- .setInstance(new TestImpl2()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- TestProtocol4 proxy = RPC.getProxy(TestProtocol4.class,
- TestProtocol4.versionID, addr, conf);
- try {
- proxy.echo(21);
- fail("The call must throw VersionMismatch exception");
- } catch (RemoteException ex) {
- Assert.assertEquals(RPC.VersionMismatch.class.getName(),
- ex.getClassName());
- Assert.assertTrue(ex.getErrorCode().equals(
- RpcErrorCodeProto.ERROR_RPC_VERSION_MISMATCH));
- } catch (IOException ex) {
- fail("Expected version mismatch but got " + ex);
- }
- }
-
- @Test
- public void testIsMethodSupported() throws IOException {
- server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
- .setInstance(new TestImpl2()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- TestProtocol2 proxy = RPC.getProxy(TestProtocol2.class,
- TestProtocol2.versionID, addr, conf);
- boolean supported = RpcClientUtil.isMethodSupported(proxy,
- TestProtocol2.class, RPC.RpcKind.RPC_WRITABLE,
- RPC.getProtocolVersion(TestProtocol2.class), "echo");
- Assert.assertTrue(supported);
- supported = RpcClientUtil.isMethodSupported(proxy,
- TestProtocol2.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
- RPC.getProtocolVersion(TestProtocol2.class), "echo");
- Assert.assertFalse(supported);
- }
-
- /**
- * Verify that ProtocolMetaInfoServerSideTranslatorPB correctly looks up
- * the server registry to extract protocol signatures and versions.
- */
- @Test
- public void testProtocolMetaInfoSSTranslatorPB() throws Exception {
- TestImpl1 impl = new TestImpl1();
- server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
- .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
- .setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
- server.start();
-
- ProtocolMetaInfoServerSideTranslatorPB xlator =
- new ProtocolMetaInfoServerSideTranslatorPB(server);
-
- GetProtocolSignatureResponseProto resp = xlator.getProtocolSignature(
- null,
- createGetProtocolSigRequestProto(TestProtocol1.class,
- RPC.RpcKind.RPC_PROTOCOL_BUFFER));
- //No signatures should be found
- Assert.assertEquals(0, resp.getProtocolSignatureCount());
- resp = xlator.getProtocolSignature(
- null,
- createGetProtocolSigRequestProto(TestProtocol1.class,
- RPC.RpcKind.RPC_WRITABLE));
- Assert.assertEquals(1, resp.getProtocolSignatureCount());
- ProtocolSignatureProto sig = resp.getProtocolSignatureList().get(0);
- Assert.assertEquals(TestProtocol1.versionID, sig.getVersion());
- boolean found = false;
- int expected = ProtocolSignature.getFingerprint(TestProtocol1.class
- .getMethod("echo", String.class));
- for (int m : sig.getMethodsList()) {
- if (expected == m) {
- found = true;
- break;
- }
- }
- Assert.assertTrue(found);
- }
-
- private GetProtocolSignatureRequestProto createGetProtocolSigRequestProto(
- Class<?> protocol, RPC.RpcKind rpcKind) {
- GetProtocolSignatureRequestProto.Builder builder =
- GetProtocolSignatureRequestProto.newBuilder();
- builder.setProtocol(protocol.getName());
- builder.setRpcKind(rpcKind.toString());
- return builder.build();
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
index 5807998..b22f91b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.ipc;
import org.apache.hadoop.conf.Configuration;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
-import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -30,11 +28,13 @@ import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.nio.channels.ClosedByInterruptException;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY;
+
/**
* tests that the proxy can be interrupted
*/
-public class TestRPCWaitForProxy extends Assert {
- private static final String ADDRESS = "0.0.0.0";
+public class TestRPCWaitForProxy extends TestRpcBase {
private static final Logger
LOG = LoggerFactory.getLogger(TestRPCWaitForProxy.class);
@@ -46,14 +46,15 @@ public class TestRPCWaitForProxy extends Assert {
*
* @throws Throwable any exception other than that which was expected
*/
- @Test(timeout = 10000)
+ @Test(timeout = 50000)
public void testWaitForProxy() throws Throwable {
RpcThread worker = new RpcThread(0);
worker.start();
worker.join();
Throwable caught = worker.getCaught();
- assertNotNull("No exception was raised", caught);
- if (!(caught instanceof ConnectException)) {
+ Throwable cause = caught.getCause();
+ Assert.assertNotNull("No exception was raised", cause);
+ if (!(cause instanceof ConnectException)) {
throw caught;
}
}
@@ -69,11 +70,11 @@ public class TestRPCWaitForProxy extends Assert {
RpcThread worker = new RpcThread(100);
worker.start();
Thread.sleep(1000);
- assertTrue("worker hasn't started", worker.waitStarted);
+ Assert.assertTrue("worker hasn't started", worker.waitStarted);
worker.interrupt();
worker.join();
Throwable caught = worker.getCaught();
- assertNotNull("No exception was raised", caught);
+ Assert.assertNotNull("No exception was raised", caught);
// looking for the root cause here, which can be wrapped
// as part of the NetUtils work. Having this test look
// a the type of exception there would be brittle to improvements
@@ -82,6 +83,8 @@ public class TestRPCWaitForProxy extends Assert {
if (cause == null) {
// no inner cause, use outer exception as root cause.
cause = caught;
+ } else if (cause.getCause() != null) {
+ cause = cause.getCause();
}
if (!(cause instanceof InterruptedIOException)
&& !(cause instanceof ClosedByInterruptException)) {
@@ -112,12 +115,16 @@ public class TestRPCWaitForProxy extends Assert {
IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
connectRetries);
waitStarted = true;
- TestProtocol proxy = RPC.waitForProxy(TestProtocol.class,
- TestProtocol.versionID,
- new InetSocketAddress(ADDRESS, 20),
- config,
- 15000L);
- proxy.echo("");
+
+ short invalidPort = 20;
+ InetSocketAddress invalidAddress = new InetSocketAddress(ADDRESS,
+ invalidPort);
+ TestRpcBase.TestRpcService proxy = RPC.getProxy(
+ TestRpcBase.TestRpcService.class,
+ 1L, invalidAddress, conf);
+ // Test echo method
+ proxy.echo(null, newEchoRequest("hello"));
+
} catch (Throwable throwable) {
caught = throwable;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
index bc604a4..5a8f8d0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
@@ -112,7 +112,8 @@ public class TestRpcBase {
return setupTestServer(builder);
}
- protected static RPC.Server setupTestServer(RPC.Builder builder) throws IOException {
+ protected static RPC.Server setupTestServer(
+ RPC.Builder builder) throws IOException {
RPC.Server server = builder.build();
server.start();
@@ -175,17 +176,21 @@ public class TestRpcBase {
public TestTokenIdentifier() {
this(new Text(), new Text());
}
+
public TestTokenIdentifier(Text tokenid) {
this(tokenid, new Text());
}
+
public TestTokenIdentifier(Text tokenid, Text realUser) {
this.tokenid = tokenid == null ? new Text() : tokenid;
this.realUser = realUser == null ? new Text() : realUser;
}
+
@Override
public Text getKind() {
return KIND_NAME;
}
+
@Override
public UserGroupInformation getUser() {
if (realUser.toString().isEmpty()) {
@@ -203,6 +208,7 @@ public class TestRpcBase {
tokenid.readFields(in);
realUser.readFields(in);
}
+
@Override
public void write(DataOutput out) throws IOException {
tokenid.write(out);
@@ -234,7 +240,7 @@ public class TestRpcBase {
@SuppressWarnings("unchecked")
@Override
public Token<TestTokenIdentifier> selectToken(Text service,
- Collection<Token<? extends TokenIdentifier>> tokens) {
+ Collection<Token<? extends TokenIdentifier>> tokens) {
if (service == null) {
return null;
}
@@ -388,19 +394,17 @@ public class TestRpcBase {
}
@Override
- public TestProtos.AuthUserResponseProto getAuthUser(
+ public TestProtos.UserResponseProto getAuthUser(
RpcController controller, TestProtos.EmptyRequestProto request)
throws ServiceException {
- UserGroupInformation authUser = null;
+ UserGroupInformation authUser;
try {
authUser = UserGroupInformation.getCurrentUser();
} catch (IOException e) {
throw new ServiceException(e);
}
- return TestProtos.AuthUserResponseProto.newBuilder()
- .setAuthUser(authUser.getUserName())
- .build();
+ return newUserResponse(authUser.getUserName());
}
@Override
@@ -432,6 +436,34 @@ public class TestRpcBase {
return TestProtos.EmptyResponseProto.newBuilder().build();
}
+
+ @Override
+ public TestProtos.UserResponseProto getCurrentUser(
+ RpcController controller,
+ TestProtos.EmptyRequestProto request) throws ServiceException {
+ String user;
+ try {
+ user = UserGroupInformation.getCurrentUser().toString();
+ } catch (IOException e) {
+ throw new ServiceException("Failed to get current user", e);
+ }
+
+ return newUserResponse(user);
+ }
+
+ @Override
+ public TestProtos.UserResponseProto getServerRemoteUser(
+ RpcController controller,
+ TestProtos.EmptyRequestProto request) throws ServiceException {
+ String serverRemoteUser = Server.getRemoteUser().toString();
+ return newUserResponse(serverRemoteUser);
+ }
+
+ private TestProtos.UserResponseProto newUserResponse(String user) {
+ return TestProtos.UserResponseProto.newBuilder()
+ .setUser(user)
+ .build();
+ }
}
protected static TestProtos.EmptyRequestProto newEmptyRequest() {
@@ -478,8 +510,4 @@ public class TestRpcBase {
}
return null;
}
-
- protected static String convert(TestProtos.AuthUserResponseProto response) {
- return response.getAuthUser();
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
index 72371a7..c48ff2e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
@@ -45,30 +45,55 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import javax.security.auth.callback.*;
-import javax.security.sasl.*;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.NameCallback;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.auth.callback.UnsupportedCallbackException;
+import javax.security.sasl.AuthorizeCallback;
+import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslClient;
+import javax.security.sasl.SaslException;
+import javax.security.sasl.SaslServer;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.security.Security;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_RPC_PROTECTION;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.*;
-import static org.junit.Assert.*;
+import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.KERBEROS;
+import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.SIMPLE;
+import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.TOKEN;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/** Unit tests for using Sasl over RPC. */
@RunWith(Parameterized.class)
public class TestSaslRPC extends TestRpcBase {
@Parameters
public static Collection<Object[]> data() {
- Collection<Object[]> params = new ArrayList<Object[]>();
+ Collection<Object[]> params = new ArrayList<>();
for (QualityOfProtection qop : QualityOfProtection.values()) {
params.add(new Object[]{ new QualityOfProtection[]{qop},qop, null });
}
@@ -114,7 +139,7 @@ public class TestSaslRPC extends TestRpcBase {
NONE(),
VALID(),
INVALID(),
- OTHER();
+ OTHER()
}
@BeforeClass
@@ -230,7 +255,7 @@ public class TestSaslRPC extends TestRpcBase {
final Server server = setupTestServer(conf, 5, sm);
doDigestRpc(server, sm);
} finally {
- SecurityUtil.setSecurityInfoProviders(new SecurityInfo[0]);
+ SecurityUtil.setSecurityInfoProviders();
}
}
@@ -259,7 +284,7 @@ public class TestSaslRPC extends TestRpcBase {
addr = NetUtils.getConnectAddress(server);
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()));
- Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId, sm);
+ Token<TestTokenIdentifier> token = new Token<>(tokenId, sm);
SecurityUtil.setTokenService(token, addr);
current.addToken(token);
@@ -296,8 +321,8 @@ public class TestSaslRPC extends TestRpcBase {
// set doPing to true
newConf.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true);
- ConnectionId remoteId = ConnectionId.getConnectionId(new InetSocketAddress(0),
- TestRpcService.class, null, 0, null, newConf);
+ ConnectionId remoteId = ConnectionId.getConnectionId(
+ new InetSocketAddress(0), TestRpcService.class, null, 0, null, newConf);
assertEquals(CommonConfigurationKeys.IPC_PING_INTERVAL_DEFAULT,
remoteId.getPingInterval());
// set doPing to false
@@ -806,13 +831,13 @@ public class TestSaslRPC extends TestRpcBase {
final TestTokenSecretManager sm = new TestTokenSecretManager();
boolean useSecretManager = (serverAuth != SIMPLE);
if (enableSecretManager != null) {
- useSecretManager &= enableSecretManager.booleanValue();
+ useSecretManager &= enableSecretManager;
}
if (forceSecretManager != null) {
- useSecretManager |= forceSecretManager.booleanValue();
+ useSecretManager |= forceSecretManager;
}
final SecretManager<?> serverSm = useSecretManager ? sm : null;
-
+
Server server = serverUgi.doAs(new PrivilegedExceptionAction<Server>() {
@Override
public Server run() throws IOException {
@@ -867,13 +892,13 @@ public class TestSaslRPC extends TestRpcBase {
proxy.ping(null, newEmptyRequest());
// make sure the other side thinks we are who we said we are!!!
assertEquals(clientUgi.getUserName(),
- convert(proxy.getAuthUser(null, newEmptyRequest())));
+ proxy.getAuthUser(null, newEmptyRequest()).getUser());
AuthMethod authMethod =
convert(proxy.getAuthMethod(null, newEmptyRequest()));
// verify sasl completed with correct QOP
assertEquals((authMethod != SIMPLE) ? expectedQop.saslQop : null,
- RPC.getConnectionIdForProxy(proxy).getSaslQop());
- return authMethod.toString();
+ RPC.getConnectionIdForProxy(proxy).getSaslQop());
+ return authMethod != null ? authMethod.toString() : null;
} catch (ServiceException se) {
if (se.getCause() instanceof RemoteException) {
throw (RemoteException) se.getCause();
@@ -898,21 +923,18 @@ public class TestSaslRPC extends TestRpcBase {
String actual) {
assertEquals(expect.toString(), actual);
}
-
- private static void assertAuthEquals(Pattern expect,
- String actual) {
+
+ private static void assertAuthEquals(Pattern expect, String actual) {
// this allows us to see the regexp and the value it didn't match
if (!expect.matcher(actual).matches()) {
- assertEquals(expect, actual); // it failed
- } else {
- assertTrue(true); // it matched
+ fail(); // it failed
}
}
/*
* Class used to test overriding QOP values using SaslPropertiesResolver
*/
- static class AuthSaslPropertiesResolver extends SaslPropertiesResolver{
+ static class AuthSaslPropertiesResolver extends SaslPropertiesResolver {
@Override
public Map<String, String> getServerProperties(InetAddress address) {
@@ -921,7 +943,7 @@ public class TestSaslRPC extends TestRpcBase {
return newPropertes;
}
}
-
+
public static void main(String[] args) throws Exception {
System.out.println("Testing Kerberos authentication over RPC");
if (args.length != 2) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
index 50d389c..c4dbcac 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
@@ -17,40 +17,35 @@
*/
package org.apache.hadoop.security;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.NetworkInterface;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Enumeration;
-
-import org.junit.Assert;
-
+import com.google.protobuf.ServiceException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.ipc.VersionedProtocol;
-import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ipc.TestRpcBase;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenInfo;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.apache.hadoop.ipc.TestRpcBase.TestTokenSecretManager;
-import org.apache.hadoop.ipc.TestRpcBase.TestTokenIdentifier;
-import org.apache.hadoop.ipc.TestRpcBase.TestTokenSelector;
-import org.apache.commons.logging.*;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Enumeration;
/**
- *
+ * Test do as effective user.
*/
-public class TestDoAsEffectiveUser {
+public class TestDoAsEffectiveUser extends TestRpcBase {
final private static String REAL_USER_NAME = "realUser1@HADOOP.APACHE.ORG";
final private static String REAL_USER_SHORT_NAME = "realUser1";
final private static String PROXY_USER_NAME = "proxyUser";
@@ -58,8 +53,8 @@ public class TestDoAsEffectiveUser {
final private static String GROUP2_NAME = "group2";
final private static String[] GROUP_NAMES = new String[] { GROUP1_NAME,
GROUP2_NAME };
- private static final String ADDRESS = "0.0.0.0";
- private TestProtocol proxy;
+
+ private TestRpcService client;
private static final Configuration masterConf = new Configuration();
@@ -82,7 +77,7 @@ public class TestDoAsEffectiveUser {
private void configureSuperUserIPAddresses(Configuration conf,
String superUserShortName) throws IOException {
- ArrayList<String> ipList = new ArrayList<String>();
+ ArrayList<String> ipList = new ArrayList<>();
Enumeration<NetworkInterface> netInterfaceList = NetworkInterface
.getNetworkInterfaces();
while (netInterfaceList.hasMoreElements()) {
@@ -130,50 +125,19 @@ public class TestDoAsEffectiveUser {
curUGI.toString());
}
- @TokenInfo(TestTokenSelector.class)
- public interface TestProtocol extends VersionedProtocol {
- public static final long versionID = 1L;
-
- String aMethod() throws IOException;
- String getServerRemoteUser() throws IOException;
- }
-
- public class TestImpl implements TestProtocol {
-
- @Override
- public String aMethod() throws IOException {
- return UserGroupInformation.getCurrentUser().toString();
- }
-
- @Override
- public String getServerRemoteUser() throws IOException {
- return Server.getRemoteUser().toString();
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return TestProtocol.versionID;
- }
-
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- return new ProtocolSignature(TestProtocol.versionID, null);
- }
- }
-
- private void checkRemoteUgi(final Server server,
- final UserGroupInformation ugi, final Configuration conf)
- throws Exception {
+ private void checkRemoteUgi(final UserGroupInformation ugi,
+ final Configuration conf) throws Exception {
ugi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
- public Void run() throws IOException {
- proxy = RPC.getProxy(
- TestProtocol.class, TestProtocol.versionID,
- NetUtils.getConnectAddress(server), conf);
- Assert.assertEquals(ugi.toString(), proxy.aMethod());
- Assert.assertEquals(ugi.toString(), proxy.getServerRemoteUser());
+ public Void run() throws ServiceException {
+ client = getClient(addr, conf);
+ String currentUser = client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
+ String serverRemoteUser = client.getServerRemoteUser(null,
+ newEmptyRequest()).getUser();
+
+ Assert.assertEquals(ugi.toString(), currentUser);
+ Assert.assertEquals(ugi.toString(), serverRemoteUser);
return null;
}
});
@@ -185,29 +149,27 @@ public class TestDoAsEffectiveUser {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1");
configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(5).setVerbose(true).build();
+ // Set RPC engine to protobuf RPC engine
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 5);
refreshConf(conf);
try {
- server.start();
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
- checkRemoteUgi(server, realUserUgi, conf);
+ checkRemoteUgi(realUserUgi, conf);
- UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting(
+ UserGroupInformation proxyUserUgi =
+ UserGroupInformation.createProxyUserForTesting(
PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
- checkRemoteUgi(server, proxyUserUgi, conf);
+ checkRemoteUgi(proxyUserUgi, conf);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -218,29 +180,25 @@ public class TestDoAsEffectiveUser {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group1");
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 5);
refreshConf(conf);
try {
- server.start();
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
- checkRemoteUgi(server, realUserUgi, conf);
+ checkRemoteUgi(realUserUgi, conf);
UserGroupInformation proxyUserUgi = UserGroupInformation
.createProxyUserForTesting(PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
- checkRemoteUgi(server, proxyUserUgi, conf);
+ checkRemoteUgi(proxyUserUgi, conf);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -256,17 +214,14 @@ public class TestDoAsEffectiveUser {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group1");
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 5);
refreshConf(conf);
try {
- server.start();
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -275,11 +230,10 @@ public class TestDoAsEffectiveUser {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws IOException {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ public String run() throws ServiceException {
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
}
});
@@ -287,10 +241,7 @@ public class TestDoAsEffectiveUser {
} catch (Exception e) {
e.printStackTrace();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -299,17 +250,14 @@ public class TestDoAsEffectiveUser {
final Configuration conf = new Configuration();
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1");
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 2);
refreshConf(conf);
try {
- server.start();
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -318,11 +266,10 @@ public class TestDoAsEffectiveUser {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws IOException {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ public String run() throws ServiceException {
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
}
});
@@ -330,10 +277,7 @@ public class TestDoAsEffectiveUser {
} catch (Exception e) {
e.printStackTrace();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -341,15 +285,12 @@ public class TestDoAsEffectiveUser {
public void testRealUserGroupNotSpecified() throws IOException {
final Configuration conf = new Configuration();
configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 2);
try {
- server.start();
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -358,11 +299,10 @@ public class TestDoAsEffectiveUser {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws IOException {
- proxy = (TestProtocol) RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ public String run() throws ServiceException {
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
}
});
@@ -370,10 +310,7 @@ public class TestDoAsEffectiveUser {
} catch (Exception e) {
e.printStackTrace();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -384,17 +321,14 @@ public class TestDoAsEffectiveUser {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group3");
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 2);
refreshConf(conf);
try {
- server.start();
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -403,11 +337,10 @@ public class TestDoAsEffectiveUser {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws IOException {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ public String run() throws ServiceException {
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
}
});
@@ -415,10 +348,7 @@ public class TestDoAsEffectiveUser {
} catch (Exception e) {
e.printStackTrace();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -432,20 +362,17 @@ public class TestDoAsEffectiveUser {
final Configuration conf = new Configuration(masterConf);
TestTokenSecretManager sm = new TestTokenSecretManager();
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
UserGroupInformation.setConfiguration(conf);
- final Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(5).setVerbose(true).setSecretManager(sm).build();
-
- server.start();
+ final Server server = setupTestServer(conf, 5, sm);
final UserGroupInformation current = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()), new Text("SomeSuperUser"));
- Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
+ Token<TestTokenIdentifier> token = new Token<>(tokenId,
sm);
SecurityUtil.setTokenService(token, addr);
UserGroupInformation proxyUserUgi = UserGroupInformation
@@ -453,23 +380,19 @@ public class TestDoAsEffectiveUser {
proxyUserUgi.addToken(token);
refreshConf(conf);
-
+
String retVal = proxyUserUgi.doAs(new PrivilegedExceptionAction<String>() {
@Override
public String run() throws Exception {
try {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
});
@@ -486,42 +409,34 @@ public class TestDoAsEffectiveUser {
TestTokenSecretManager sm = new TestTokenSecretManager();
final Configuration newConf = new Configuration(masterConf);
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, newConf);
+ // Set RPC engine to protobuf RPC engine
+ RPC.setProtocolEngine(newConf, TestRpcService.class,
+ ProtobufRpcEngine.class);
UserGroupInformation.setConfiguration(newConf);
- final Server server = new RPC.Builder(newConf)
- .setProtocol(TestProtocol.class).setInstance(new TestImpl())
- .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
- .setSecretManager(sm).build();
-
- server.start();
+ final Server server = setupTestServer(newConf, 5, sm);
final UserGroupInformation current = UserGroupInformation
.createUserForTesting(REAL_USER_NAME, GROUP_NAMES);
refreshConf(newConf);
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()), new Text("SomeSuperUser"));
- Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
- sm);
+ Token<TestTokenIdentifier> token = new Token<>(tokenId, sm);
SecurityUtil.setTokenService(token, addr);
current.addToken(token);
String retVal = current.doAs(new PrivilegedExceptionAction<String>() {
@Override
public String run() throws Exception {
try {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, newConf);
- String ret = proxy.aMethod();
- return ret;
+ client = getClient(addr, newConf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index ef5c6a2..a306d35 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.TestRpcBase.TestTokenIdentifier;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -28,7 +29,11 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
-import org.junit.*;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
@@ -49,9 +54,22 @@ import java.util.Set;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
-import static org.apache.hadoop.ipc.TestSaslRPC.*;
-import static org.apache.hadoop.test.MetricsAsserts.*;
-import static org.junit.Assert.*;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt;
+import static org.apache.hadoop.test.MetricsAsserts.assertGaugeGt;
+import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges;
+import static org.apache.hadoop.test.MetricsAsserts.getDoubleGauge;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -108,7 +126,7 @@ public class TestUserGroupInformation {
UserGroupInformation.setLoginUser(null);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testSimpleLogin() throws IOException {
tryLoginAuthenticationMethod(AuthenticationMethod.SIMPLE, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/proto/test.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test.proto b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
index 99cd93d..6411f97 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
@@ -88,6 +88,6 @@ message AuthMethodResponseProto {
required string mechanismName = 2;
}
-message AuthUserResponseProto {
- required string authUser = 1;
+message UserResponseProto {
+ required string user = 1;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
index 3292115..06f6c4f 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
@@ -40,9 +40,11 @@ service TestProtobufRpcProto {
rpc exchange(ExchangeRequestProto) returns (ExchangeResponseProto);
rpc sleep(SleepRequestProto) returns (EmptyResponseProto);
rpc getAuthMethod(EmptyRequestProto) returns (AuthMethodResponseProto);
- rpc getAuthUser(EmptyRequestProto) returns (AuthUserResponseProto);
+ rpc getAuthUser(EmptyRequestProto) returns (UserResponseProto);
rpc echoPostponed(EchoRequestProto) returns (EchoResponseProto);
rpc sendPostponed(EmptyRequestProto) returns (EmptyResponseProto);
+ rpc getCurrentUser(EmptyRequestProto) returns (UserResponseProto);
+ rpc getServerRemoteUser(EmptyRequestProto) returns (UserResponseProto);
}
service TestProtobufRpc2Proto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 6b52949..57f7cb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -168,7 +168,6 @@ import org.apache.hadoop.ipc.RetryCache.CacheEntry;
import org.apache.hadoop.ipc.RetryCache.CacheEntryWithPayload;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.StandbyException;
-import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.ipc.RefreshRegistry;
import org.apache.hadoop.ipc.RefreshResponse;
import org.apache.hadoop.net.Node;
@@ -317,8 +316,6 @@ public class NameNodeRpcServer implements NamenodeProtocols {
new TraceAdminProtocolServerSideTranslatorPB(this);
BlockingService traceAdminService = TraceAdminService
.newReflectiveBlockingService(traceAdminXlator);
-
- WritableRpcEngine.ensureInitialized();
InetSocketAddress serviceRpcAddr = nn.getServiceRpcServerAddress(conf);
if (serviceRpcAddr != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
deleted file mode 100644
index 0b7ee33..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.security;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.mockito.Mockito.mock;
-
-import java.net.InetSocketAddress;
-import java.security.PrivilegedExceptionAction;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SaslInputStream;
-import org.apache.hadoop.security.SaslRpcClient;
-import org.apache.hadoop.security.SaslRpcServer;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
-import org.junit.Test;
-
-/** Unit tests for using Delegation Token over RPC. */
-public class TestClientProtocolWithDelegationToken {
- private static final String ADDRESS = "0.0.0.0";
-
- public static final Log LOG = LogFactory
- .getLog(TestClientProtocolWithDelegationToken.class);
-
- private static final Configuration conf;
- static {
- conf = new Configuration();
- conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- UserGroupInformation.setConfiguration(conf);
- }
-
- static {
- GenericTestUtils.setLogLevel(Client.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(Server.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(SaslRpcClient.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(SaslRpcServer.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(SaslInputStream.LOG, Level.ALL);
- }
-
- @Test
- public void testDelegationTokenRpc() throws Exception {
- ClientProtocol mockNN = mock(ClientProtocol.class);
- FSNamesystem mockNameSys = mock(FSNamesystem.class);
-
- DelegationTokenSecretManager sm = new DelegationTokenSecretManager(
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
- 3600000, mockNameSys);
- sm.startThreads();
- final Server server = new RPC.Builder(conf)
- .setProtocol(ClientProtocol.class).setInstance(mockNN)
- .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
- .setSecretManager(sm).build();
-
- server.start();
-
- final UserGroupInformation current = UserGroupInformation.getCurrentUser();
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
- String user = current.getUserName();
- Text owner = new Text(user);
- DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null);
- Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
- dtId, sm);
- SecurityUtil.setTokenService(token, addr);
- LOG.info("Service for token is " + token.getService());
- current.addToken(token);
- current.doAs(new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws Exception {
- ClientProtocol proxy = null;
- try {
- proxy = RPC.getProxy(ClientProtocol.class,
- ClientProtocol.versionID, addr, conf);
- proxy.getServerDefaults();
- } finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
- }
- return null;
- }
- });
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
index 3fef5e2..729af0a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Groups;
@@ -98,8 +97,6 @@ public class HSAdminServer extends AbstractService implements HSAdminProtocol {
BlockingService refreshHSAdminProtocolService = HSAdminRefreshProtocolService
.newReflectiveBlockingService(refreshHSAdminProtocolXlator);
- WritableRpcEngine.ensureInitialized();
-
clientRpcAddress = conf.getSocketAddr(
JHAdminConfig.MR_HISTORY_BIND_HOST,
JHAdminConfig.JHS_ADMIN_ADDRESS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/22] hadoop git commit: HADOOP-13549. Eliminate intermediate
buffer for server-side PB encoding. Contributed by Daryn Sharp.
Posted by sh...@apache.org.
HADOOP-13549. Eliminate intermediate buffer for server-side PB encoding. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39d1b1d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39d1b1d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39d1b1d7
Branch: refs/heads/HADOOP-12756
Commit: 39d1b1d747b1e325792b897b3264272f32b756a9
Parents: e6fcfe2
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Sep 6 10:01:01 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue Sep 6 10:02:42 2016 -0500
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ipc/Server.java | 52 ++++++++++++++++++--
1 file changed, 49 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39d1b1d7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index ea4f533..531d574 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -119,6 +119,7 @@ import org.apache.htrace.core.Tracer;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.Message;
/** An abstract IPC service. IPC calls take a single {@link Writable} as a
@@ -2752,24 +2753,69 @@ public abstract class Server {
private void setupResponse(RpcCall call,
RpcResponseHeaderProto header, Writable rv) throws IOException {
+ final byte[] response;
+ if (rv == null || (rv instanceof RpcWritable.ProtobufWrapper)) {
+ response = setupResponseForProtobuf(header, rv);
+ } else {
+ response = setupResponseForWritable(header, rv);
+ }
+ if (response.length > maxRespSize) {
+ LOG.warn("Large response size " + response.length + " for call "
+ + call.toString());
+ }
+ call.setResponse(ByteBuffer.wrap(response));
+ }
+
+ private byte[] setupResponseForWritable(
+ RpcResponseHeaderProto header, Writable rv) throws IOException {
ResponseBuffer buf = responseBuffer.get().reset();
try {
RpcWritable.wrap(header).writeTo(buf);
if (rv != null) {
RpcWritable.wrap(rv).writeTo(buf);
}
- call.setResponse(ByteBuffer.wrap(buf.toByteArray()));
+ return buf.toByteArray();
} finally {
// Discard a large buf and reset it back to smaller size
// to free up heap.
if (buf.capacity() > maxRespSize) {
- LOG.warn("Large response size " + buf.size() + " for call "
- + call.toString());
buf.setCapacity(INITIAL_RESP_BUF_SIZE);
}
}
}
+
+ // writing to a pre-allocated array is the most efficient way to construct
+ // a protobuf response.
+ private byte[] setupResponseForProtobuf(
+ RpcResponseHeaderProto header, Writable rv) throws IOException {
+ Message payload = (rv != null)
+ ? ((RpcWritable.ProtobufWrapper)rv).getMessage() : null;
+ int length = getDelimitedLength(header);
+ if (payload != null) {
+ length += getDelimitedLength(payload);
+ }
+ byte[] buf = new byte[length + 4];
+ CodedOutputStream cos = CodedOutputStream.newInstance(buf);
+ // the stream only supports little endian ints
+ cos.writeRawByte((byte)((length >>> 24) & 0xFF));
+ cos.writeRawByte((byte)((length >>> 16) & 0xFF));
+ cos.writeRawByte((byte)((length >>> 8) & 0xFF));
+ cos.writeRawByte((byte)((length >>> 0) & 0xFF));
+ cos.writeRawVarint32(header.getSerializedSize());
+ header.writeTo(cos);
+ if (payload != null) {
+ cos.writeRawVarint32(payload.getSerializedSize());
+ payload.writeTo(cos);
+ }
+ return buf;
+ }
+
+ private static int getDelimitedLength(Message message) {
+ int length = message.getSerializedSize();
+ return length + CodedOutputStream.computeRawVarint32Size(length);
+ }
+
/**
* Setup response for the IPC Call on Fatal Error from a
* client that is using old version of Hadoop.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/22] hadoop git commit: HDFS-6962. ACL inheritance conflicts with
umaskmode. Contributed by Chris Nauroth.
Posted by sh...@apache.org.
HDFS-6962. ACL inheritance conflicts with umaskmode. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0d5382f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0d5382f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0d5382f
Branch: refs/heads/HADOOP-12756
Commit: f0d5382ff3e31a47d13e4cb6c3a244cca82b17ce
Parents: d37dc5d
Author: Chris Nauroth <cn...@apache.org>
Authored: Tue Sep 6 11:02:39 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Tue Sep 6 11:02:39 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileContext.java | 8 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 7 +-
.../hadoop/fs/permission/FsCreateModes.java | 101 ++
.../hadoop/fs/permission/FsPermission.java | 14 +
.../hadoop/fs/permission/package-info.java | 25 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 5 +-
.../ClientNamenodeProtocolTranslatorPB.java | 14 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +-
.../hdfs/web/resources/PermissionParam.java | 9 +-
.../web/resources/UnmaskedPermissionParam.java | 51 +
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
...tNamenodeProtocolServerSideTranslatorPB.java | 16 +-
.../datanode/web/webhdfs/ParameterParser.java | 7 +
.../datanode/web/webhdfs/WebHdfsHandler.java | 6 +-
.../hadoop/hdfs/server/namenode/AclStorage.java | 7 +-
.../hdfs/server/namenode/FSDirMkdirOp.java | 3 +-
.../hdfs/server/namenode/FSDirSymlinkOp.java | 3 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 5 +-
.../hdfs/server/namenode/FSDirectory.java | 72 +-
.../web/resources/NamenodeWebHdfsMethods.java | 49 +-
.../src/main/resources/hdfs-default.xml | 14 +-
.../src/site/markdown/HdfsPermissionsGuide.md | 6 +
.../java/org/apache/hadoop/cli/TestAclCLI.java | 6 +-
.../cli/TestAclCLIWithPosixAclInheritance.java | 45 +
.../hdfs/server/namenode/FSAclBaseTest.java | 91 ++
.../server/namenode/TestGetBlockLocations.java | 2 +-
.../snapshot/TestRenameWithSnapshots.java | 4 +-
.../testAclCLIWithPosixAclInheritance.xml | 1075 ++++++++++++++++++
29 files changed, 1613 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index f235773..160a63d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
@@ -671,7 +672,7 @@ public class FileContext {
CreateOpts.Perms permOpt = CreateOpts.getOpt(CreateOpts.Perms.class, opts);
FsPermission permission = (permOpt != null) ? permOpt.getValue() :
FILE_DEFAULT_PERM;
- permission = permission.applyUMask(getUMask());
+ permission = FsCreateModes.applyUMask(permission, getUMask());
final CreateOpts[] updatedOpts =
CreateOpts.setOpt(CreateOpts.perms(permission), opts);
@@ -717,8 +718,9 @@ public class FileContext {
ParentNotDirectoryException, UnsupportedFileSystemException,
IOException {
final Path absDir = fixRelativePart(dir);
- final FsPermission absFerms = (permission == null ?
- FsPermission.getDirDefault() : permission).applyUMask(getUMask());
+ final FsPermission absFerms = FsCreateModes.applyUMask(
+ permission == null ?
+ FsPermission.getDirDefault() : permission, getUMask());
new FSLinkResolver<Void>() {
@Override
public Void next(final AbstractFileSystem fs, final Path p)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index c366729..9bde29d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.io.Text;
@@ -925,9 +926,9 @@ public abstract class FileSystem extends Configured implements Closeable {
long blockSize,
Progressable progress
) throws IOException {
- return this.create(f, FsPermission.getFileDefault().applyUMask(
- FsPermission.getUMask(getConf())), overwrite, bufferSize,
- replication, blockSize, progress);
+ return this.create(f, FsCreateModes.applyUMask(
+ FsPermission.getFileDefault(), FsPermission.getUMask(getConf())),
+ overwrite, bufferSize, replication, blockSize, progress);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
new file mode 100644
index 0000000..a1ed0d7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.permission;
+
+import java.text.MessageFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * A class that stores both masked and unmasked create modes
+ * and is a drop-in replacement for masked permission.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class FsCreateModes extends FsPermission {
+ private final FsPermission unmasked;
+
+ /**
+ * Create from unmasked mode and umask.
+ *
+ * If the mode is already an FsCreateModes object, return it.
+ */
+ public static FsPermission applyUMask(FsPermission mode,
+ FsPermission umask) {
+ if (mode.getUnmasked() != null) {
+ return mode;
+ }
+ return create(mode.applyUMask(umask), mode);
+ }
+
+ /**
+ * Create from masked and unmasked modes.
+ */
+ public static FsCreateModes create(FsPermission masked,
+ FsPermission unmasked) {
+ assert masked.getUnmasked() == null;
+ assert unmasked.getUnmasked() == null;
+ return new FsCreateModes(masked, unmasked);
+ }
+
+ private FsCreateModes(FsPermission masked, FsPermission unmasked) {
+ super(masked);
+ this.unmasked = unmasked;
+ assert masked.getUnmasked() == null;
+ assert unmasked.getUnmasked() == null;
+ }
+
+ @Override
+ public FsPermission getMasked() {
+ return this;
+ }
+
+ @Override
+ public FsPermission getUnmasked() {
+ return unmasked;
+ }
+
+ @Override
+ public String toString() {
+ return MessageFormat.format("'{' masked: {0}, unmasked: {1} '}'",
+ super.toString(), getUnmasked());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ FsCreateModes that = (FsCreateModes) o;
+ return getUnmasked().equals(that.getUnmasked());
+ }
+
+ @Override
+ public int hashCode() {
+ int result = super.hashCode();
+ result = 31 * result + getUnmasked().hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index 78255bbd..48a5b1c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -138,6 +138,20 @@ public class FsPermission implements Writable {
}
/**
+ * Get masked permission if exists.
+ */
+ public FsPermission getMasked() {
+ return null;
+ }
+
+ /**
+ * Get unmasked permission if exists.
+ */
+ public FsPermission getUnmasked() {
+ return null;
+ }
+
+ /**
* Create and initialize a {@link FsPermission} from {@link DataInput}.
*/
public static FsPermission read(DataInput in) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/package-info.java
new file mode 100644
index 0000000..6cee4ee
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package provides support for HDFS permission and ACL.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.permission;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index ad86895..b0fcba2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -1160,14 +1161,14 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
if (permission == null) {
permission = FsPermission.getFileDefault();
}
- return permission.applyUMask(dfsClientConf.getUMask());
+ return FsCreateModes.applyUMask(permission, dfsClientConf.getUMask());
}
private FsPermission applyUMaskDir(FsPermission permission) {
if (permission == null) {
permission = FsPermission.getDirDefault();
}
- return permission.applyUMask(dfsClientConf.getUMask());
+ return FsCreateModes.applyUMask(permission, dfsClientConf.getUMask());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 57f8fd6..f73abfd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -294,6 +294,10 @@ public class ClientNamenodeProtocolTranslatorPB implements
.setCreateParent(createParent)
.setReplication(replication)
.setBlockSize(blockSize);
+ FsPermission unmasked = masked.getUnmasked();
+ if (unmasked != null) {
+ builder.setUnmasked(PBHelperClient.convert(unmasked));
+ }
builder.addAllCryptoProtocolVersion(
PBHelperClient.convert(supportedVersions));
CreateRequestProto req = builder.build();
@@ -579,11 +583,15 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public boolean mkdirs(String src, FsPermission masked, boolean createParent)
throws IOException {
- MkdirsRequestProto req = MkdirsRequestProto.newBuilder()
+ MkdirsRequestProto.Builder builder = MkdirsRequestProto.newBuilder()
.setSrc(src)
.setMasked(PBHelperClient.convert(masked))
- .setCreateParent(createParent).build();
-
+ .setCreateParent(createParent);
+ FsPermission unmasked = masked.getUnmasked();
+ if (unmasked != null) {
+ builder.setUnmasked(PBHelperClient.convert(unmasked));
+ }
+ MkdirsRequestProto req = builder.build();
try {
return rpcProxy.mkdirs(null, req).getResult();
} catch (ServiceException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 968eea3..5389a02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.fs.GlobalStorageStatistics;
import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
@@ -973,7 +974,8 @@ public class WebHdfsFileSystem extends FileSystem
if (permission == null) {
permission = FsPermission.getDefault();
}
- return permission.applyUMask(FsPermission.getUMask(getConf()));
+ return FsCreateModes.applyUMask(permission,
+ FsPermission.getUMask(getConf()));
}
private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
@@ -1025,8 +1027,10 @@ public class WebHdfsFileSystem extends FileSystem
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.MKDIRS);
final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
+ final FsPermission modes = applyUMask(permission);
return new FsPathBooleanRunner(op, f,
- new PermissionParam(applyUMask(permission))
+ new PermissionParam(modes.getMasked()),
+ new UnmaskedPermissionParam(modes.getUnmasked())
).run();
}
@@ -1313,9 +1317,11 @@ public class WebHdfsFileSystem extends FileSystem
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.CREATE);
+ final FsPermission modes = applyUMask(permission);
final HttpOpParam.Op op = PutOpParam.Op.CREATE;
return new FsPathOutputStreamRunner(op, f, bufferSize,
- new PermissionParam(applyUMask(permission)),
+ new PermissionParam(modes.getMasked()),
+ new UnmaskedPermissionParam(modes.getUnmasked()),
new OverwriteParam(overwrite),
new BufferSizeParam(bufferSize),
new ReplicationParam(replication),
@@ -1331,9 +1337,11 @@ public class WebHdfsFileSystem extends FileSystem
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.CREATE_NON_RECURSIVE);
+ final FsPermission modes = applyUMask(permission);
final HttpOpParam.Op op = PutOpParam.Op.CREATE;
return new FsPathOutputStreamRunner(op, f, bufferSize,
- new PermissionParam(applyUMask(permission)),
+ new PermissionParam(modes.getMasked()),
+ new UnmaskedPermissionParam(modes.getUnmasked()),
new CreateFlagParam(flag),
new CreateParentParam(false),
new BufferSizeParam(bufferSize),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java
index 42ff69d..1598cbc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java
@@ -54,7 +54,7 @@ public class PermissionParam extends ShortParam {
* @param value the parameter value.
*/
public PermissionParam(final FsPermission value) {
- super(DOMAIN, value == null? null: value.toShort(), null, null);
+ this(DOMAIN, value == null ? null : value.toShort(), null, null);
}
/**
@@ -62,7 +62,12 @@ public class PermissionParam extends ShortParam {
* @param str a string representation of the parameter value.
*/
public PermissionParam(final String str) {
- super(DOMAIN, DOMAIN.parse(str), (short)0, (short)01777);
+ this(DOMAIN, DOMAIN.parse(str), (short)0, (short)01777);
+ }
+
+ PermissionParam(final Domain domain, final Short value, final Short min,
+ final Short max) {
+ super(domain, value, min, max);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UnmaskedPermissionParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UnmaskedPermissionParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UnmaskedPermissionParam.java
new file mode 100644
index 0000000..bbd2a5c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UnmaskedPermissionParam.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import org.apache.hadoop.fs.permission.FsPermission;
+
+/**
+ * Unmasked permission parameter, use a Short to represent a FsPermission.
+ */
+public class UnmaskedPermissionParam extends PermissionParam {
+ /** Parameter name. */
+ public static final String NAME = "unmaskedpermission";
+
+ private static final Domain DOMAIN = new Domain(NAME, 8);
+
+ /**
+ * Constructor.
+ * @param value the parameter value.
+ */
+ public UnmaskedPermissionParam(final FsPermission value) {
+ super(DOMAIN, value == null ? null : value.toShort(), null, null);
+ }
+
+ /**
+ * Constructor.
+ * @param str a string representation of the parameter value.
+ */
+ public UnmaskedPermissionParam(final String str) {
+ super(DOMAIN, DOMAIN.parse(str), (short)0, (short)01777);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 83f5828..54bed32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -79,6 +79,7 @@ message CreateRequestProto {
required uint32 replication = 6; // Short: Only 16 bits used
required uint64 blockSize = 7;
repeated CryptoProtocolVersionProto cryptoProtocolVersion = 8;
+ optional FsPermissionProto unmasked = 9;
}
message CreateResponseProto {
@@ -264,6 +265,7 @@ message MkdirsRequestProto {
required string src = 1;
required FsPermissionProto masked = 2;
required bool createParent = 3;
+ optional FsPermissionProto unmasked = 4;
}
message MkdirsResponseProto {
required bool result = 1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f2ab321..7bdfd44 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -259,6 +259,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT = "supergroup";
public static final String DFS_NAMENODE_ACLS_ENABLED_KEY = "dfs.namenode.acls.enabled";
public static final boolean DFS_NAMENODE_ACLS_ENABLED_DEFAULT = false;
+ public static final String DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY =
+ "dfs.namenode.posix.acl.inheritance.enabled";
+ public static final boolean
+ DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT = false;
public static final String DFS_NAMENODE_XATTRS_ENABLED_KEY = "dfs.namenode.xattrs.enabled";
public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
public static final String DFS_ADMIN = "dfs.cluster.administrators";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 641cd20..3974956 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -25,6 +25,8 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
+import org.apache.hadoop.fs.permission.FsCreateModes;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.AddBlockFlag;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
@@ -411,8 +413,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public CreateResponseProto create(RpcController controller,
CreateRequestProto req) throws ServiceException {
try {
+ FsPermission masked = req.hasUnmasked() ?
+ FsCreateModes.create(PBHelperClient.convert(req.getMasked()),
+ PBHelperClient.convert(req.getUnmasked())) :
+ PBHelperClient.convert(req.getMasked());
HdfsFileStatus result = server.create(req.getSrc(),
- PBHelperClient.convert(req.getMasked()), req.getClientName(),
+ masked, req.getClientName(),
PBHelperClient.convertCreateFlag(req.getCreateFlag()), req.getCreateParent(),
(short) req.getReplication(), req.getBlockSize(),
PBHelperClient.convertCryptoProtocolVersions(
@@ -651,8 +657,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public MkdirsResponseProto mkdirs(RpcController controller,
MkdirsRequestProto req) throws ServiceException {
try {
- boolean result = server.mkdirs(req.getSrc(),
- PBHelperClient.convert(req.getMasked()), req.getCreateParent());
+ FsPermission masked = req.hasUnmasked() ?
+ FsCreateModes.create(PBHelperClient.convert(req.getMasked()),
+ PBHelperClient.convert(req.getUnmasked())) :
+ PBHelperClient.convert(req.getMasked());
+ boolean result = server.mkdirs(req.getSrc(), masked,
+ req.getCreateParent());
return MkdirsResponseProto.newBuilder().setResult(result).build();
} catch (IOException e) {
throw new ServiceException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
index 3c7016e..bb28110 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.web.resources.OffsetParam;
import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
import org.apache.hadoop.hdfs.web.resources.PermissionParam;
import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
+import org.apache.hadoop.hdfs.web.resources.UnmaskedPermissionParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.token.Token;
@@ -108,6 +109,12 @@ class ParameterParser {
getFileFsPermission();
}
+ FsPermission unmaskedPermission() {
+ String value = param(UnmaskedPermissionParam.NAME);
+ return value == null ? null :
+ new UnmaskedPermissionParam(value).getFileFsPermission();
+ }
+
boolean overwrite() {
return new OverwriteParam(param(OverwriteParam.NAME)).getValue();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index d354612..20e0a2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -66,6 +66,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
@@ -187,7 +188,10 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
final int bufferSize = params.bufferSize();
final short replication = params.replication();
final long blockSize = params.blockSize();
- final FsPermission permission = params.permission();
+ final FsPermission unmaskedPermission = params.unmaskedPermission();
+ final FsPermission permission = unmaskedPermission == null ?
+ params.permission() :
+ FsCreateModes.create(params.permission(), unmaskedPermission);
final boolean createParent = params.createParent();
EnumSet<CreateFlag> flags = params.createFlag();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
index abd3755..58dfe12 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
@@ -73,11 +73,11 @@ public final class AclStorage {
*
* @param child INode newly created child
*/
- public static void copyINodeDefaultAcl(INode child) {
+ public static boolean copyINodeDefaultAcl(INode child) {
INodeDirectory parent = child.getParent();
AclFeature parentAclFeature = parent.getAclFeature();
if (parentAclFeature == null || !(child.isFile() || child.isDirectory())) {
- return;
+ return false;
}
// Split parent's entries into access vs. default.
@@ -88,7 +88,7 @@ public final class AclStorage {
// The parent may have an access ACL but no default ACL. If so, exit.
if (parentDefaultEntries.isEmpty()) {
- return;
+ return false;
}
// Pre-allocate list size for access entries to copy from parent.
@@ -145,6 +145,7 @@ public final class AclStorage {
}
child.setPermission(newPerm);
+ return true;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index bf5ff00..2d1914f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -219,7 +219,8 @@ class FSDirMkdirOp {
final INodeDirectory dir = new INodeDirectory(inodeId, name, permission,
timestamp);
- INodesInPath iip = fsd.addLastINode(parent, dir, true);
+ INodesInPath iip =
+ fsd.addLastINode(parent, dir, permission.getPermission(), true);
if (iip != null && aclEntries != null) {
AclStorage.updateINodeAcl(dir, aclEntries, Snapshot.CURRENT_STATE_ID);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
index ec93952..6938a84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
@@ -88,7 +88,8 @@ class FSDirSymlinkOp {
final INodeSymlink symlink = new INodeSymlink(id, null, perm, mtime, atime,
target);
symlink.setLocalName(localName);
- return fsd.addINode(iip, symlink) != null ? symlink : null;
+ return fsd.addINode(iip, symlink, perm.getPermission()) != null ?
+ symlink : null;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 077f04f..aa2be92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -496,7 +496,8 @@ class FSDirWriteFileOp {
replication, preferredBlockSize, storagePolicyId, ecPolicy != null);
}
newNode.setLocalName(localName);
- INodesInPath iip = fsd.addINode(existing, newNode);
+ INodesInPath iip = fsd.addINode(existing, newNode,
+ permissions.getPermission());
if (iip != null) {
if (aclEntries != null) {
AclStorage.updateINodeAcl(newNode, aclEntries, CURRENT_STATE_ID);
@@ -594,7 +595,7 @@ class FSDirWriteFileOp {
modTime, modTime, replication, preferredBlockSize, ecPolicy != null);
newNode.setLocalName(localName);
newNode.toUnderConstruction(clientName, clientMachine);
- newiip = fsd.addINode(existing, newNode);
+ newiip = fsd.addINode(existing, newNode, permissions.getPermission());
} finally {
fsd.writeUnlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 5dc5a03..340d491 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -168,6 +168,10 @@ public class FSDirectory implements Closeable {
* ACL-related operations.
*/
private final boolean aclsEnabled;
+ /**
+ * Support for POSIX ACL inheritance. Not final for testing purpose.
+ */
+ private boolean posixAclInheritanceEnabled;
private final boolean xattrsEnabled;
private final int xattrMaxSize;
@@ -251,6 +255,10 @@ public class FSDirectory implements Closeable {
DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,
DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_DEFAULT);
LOG.info("ACLs enabled? " + aclsEnabled);
+ this.posixAclInheritanceEnabled = conf.getBoolean(
+ DFSConfigKeys.DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY,
+ DFSConfigKeys.DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT);
+ LOG.info("POSIX ACL inheritance enabled? " + posixAclInheritanceEnabled);
this.xattrsEnabled = conf.getBoolean(
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT);
@@ -450,6 +458,18 @@ public class FSDirectory implements Closeable {
boolean isAclsEnabled() {
return aclsEnabled;
}
+
+ @VisibleForTesting
+ public boolean isPosixAclInheritanceEnabled() {
+ return posixAclInheritanceEnabled;
+ }
+
+ @VisibleForTesting
+ public void setPosixAclInheritanceEnabled(
+ boolean posixAclInheritanceEnabled) {
+ this.posixAclInheritanceEnabled = posixAclInheritanceEnabled;
+ }
+
boolean isXattrsEnabled() {
return xattrsEnabled;
}
@@ -952,16 +972,18 @@ public class FSDirectory implements Closeable {
* Add the given child to the namespace.
* @param existing the INodesInPath containing all the ancestral INodes
* @param child the new INode to add
+ * @param modes create modes
* @return a new INodesInPath instance containing the new child INode. Null
* if the adding fails.
* @throws QuotaExceededException is thrown if it violates quota limit
*/
- INodesInPath addINode(INodesInPath existing, INode child)
+ INodesInPath addINode(INodesInPath existing, INode child,
+ FsPermission modes)
throws QuotaExceededException, UnresolvedLinkException {
cacheName(child);
writeLock();
try {
- return addLastINode(existing, child, true);
+ return addLastINode(existing, child, modes, true);
} finally {
writeUnlock();
}
@@ -1067,12 +1089,51 @@ public class FSDirectory implements Closeable {
}
/**
+ * Turn on HDFS-6962 POSIX ACL inheritance when the property
+ * {@link DFSConfigKeys#DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY} is
+ * true and a compatible client has sent both masked and unmasked create
+ * modes.
+ *
+ * @param child INode newly created child
+ * @param modes create modes
+ */
+ private void copyINodeDefaultAcl(INode child, FsPermission modes) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("child: {}, posixAclInheritanceEnabled: {}, modes: {}",
+ child, posixAclInheritanceEnabled, modes);
+ }
+
+ if (posixAclInheritanceEnabled && modes != null &&
+ modes.getUnmasked() != null) {
+ //
+ // HDFS-6962: POSIX ACL inheritance
+ //
+ child.setPermission(modes.getUnmasked());
+ if (!AclStorage.copyINodeDefaultAcl(child)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{}: no parent default ACL to inherit", child);
+ }
+ child.setPermission(modes.getMasked());
+ }
+ } else {
+ //
+ // Old behavior before HDFS-6962
+ //
+ AclStorage.copyINodeDefaultAcl(child);
+ }
+ }
+
+ /**
* Add a child to the end of the path specified by INodesInPath.
+ * @param existing the INodesInPath containing all the ancestral INodes
+ * @param inode the new INode to add
+ * @param modes create modes
+ * @param checkQuota whether to check quota
* @return an INodesInPath instance containing the new INode
*/
@VisibleForTesting
public INodesInPath addLastINode(INodesInPath existing, INode inode,
- boolean checkQuota) throws QuotaExceededException {
+ FsPermission modes, boolean checkQuota) throws QuotaExceededException {
assert existing.getLastINode() != null &&
existing.getLastINode().isDirectory();
@@ -1119,7 +1180,7 @@ public class FSDirectory implements Closeable {
return null;
} else {
if (!isRename) {
- AclStorage.copyINodeDefaultAcl(inode);
+ copyINodeDefaultAcl(inode, modes);
}
addToInodeMap(inode);
}
@@ -1128,7 +1189,8 @@ public class FSDirectory implements Closeable {
INodesInPath addLastINodeNoQuotaCheck(INodesInPath existing, INode i) {
try {
- return addLastINode(existing, i, false);
+ // All callers do not have create modes to pass.
+ return addLastINode(existing, i, null, false);
} catch (QuotaExceededException e) {
NameNode.LOG.warn("FSDirectory.addChildNoQuotaCheck - unexpected", e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index e3a5bad..3ab0c67 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -58,6 +58,8 @@ import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsCreateModes;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -324,6 +326,9 @@ public class NamenodeWebHdfsMethods {
final GroupParam group,
@QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
final PermissionParam permission,
+ @QueryParam(UnmaskedPermissionParam.NAME)
+ @DefaultValue(UnmaskedPermissionParam.DEFAULT)
+ final UnmaskedPermissionParam unmaskedPermission,
@QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
final OverwriteParam overwrite,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
@@ -362,11 +367,11 @@ public class NamenodeWebHdfsMethods {
final NoRedirectParam noredirect
) throws IOException, InterruptedException {
return put(ugi, delegation, username, doAsUser, ROOT, op, destination,
- owner, group, permission, overwrite, bufferSize, replication,
- blockSize, modificationTime, accessTime, renameOptions, createParent,
- delegationTokenArgument, aclPermission, xattrName, xattrValue,
- xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes,
- createFlagParam, noredirect);
+ owner, group, permission, unmaskedPermission, overwrite, bufferSize,
+ replication, blockSize, modificationTime, accessTime, renameOptions,
+ createParent, delegationTokenArgument, aclPermission, xattrName,
+ xattrValue, xattrSetFlag, snapshotName, oldSnapshotName,
+ excludeDatanodes, createFlagParam, noredirect);
}
/** Handle HTTP PUT request. */
@@ -393,6 +398,9 @@ public class NamenodeWebHdfsMethods {
final GroupParam group,
@QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
final PermissionParam permission,
+ @QueryParam(UnmaskedPermissionParam.NAME)
+ @DefaultValue(UnmaskedPermissionParam.DEFAULT)
+ final UnmaskedPermissionParam unmaskedPermission,
@QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
final OverwriteParam overwrite,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
@@ -432,10 +440,11 @@ public class NamenodeWebHdfsMethods {
) throws IOException, InterruptedException {
init(ugi, delegation, username, doAsUser, path, op, destination, owner,
- group, permission, overwrite, bufferSize, replication, blockSize,
- modificationTime, accessTime, renameOptions, delegationTokenArgument,
- aclPermission, xattrName, xattrValue, xattrSetFlag, snapshotName,
- oldSnapshotName, excludeDatanodes, createFlagParam, noredirect);
+ group, permission, unmaskedPermission, overwrite, bufferSize,
+ replication, blockSize, modificationTime, accessTime, renameOptions,
+ delegationTokenArgument, aclPermission, xattrName, xattrValue,
+ xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes,
+ createFlagParam, noredirect);
return ugi.doAs(new PrivilegedExceptionAction<Response>() {
@Override
@@ -443,10 +452,11 @@ public class NamenodeWebHdfsMethods {
try {
return put(ugi, delegation, username, doAsUser,
path.getAbsolutePath(), op, destination, owner, group,
- permission, overwrite, bufferSize, replication, blockSize,
- modificationTime, accessTime, renameOptions, createParent,
- delegationTokenArgument, aclPermission, xattrName, xattrValue,
- xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes,
+ permission, unmaskedPermission, overwrite, bufferSize,
+ replication, blockSize, modificationTime, accessTime,
+ renameOptions, createParent, delegationTokenArgument,
+ aclPermission, xattrName, xattrValue, xattrSetFlag,
+ snapshotName, oldSnapshotName, excludeDatanodes,
createFlagParam, noredirect);
} finally {
reset();
@@ -466,6 +476,7 @@ public class NamenodeWebHdfsMethods {
final OwnerParam owner,
final GroupParam group,
final PermissionParam permission,
+ final UnmaskedPermissionParam unmaskedPermission,
final OverwriteParam overwrite,
final BufferSizeParam bufferSize,
final ReplicationParam replication,
@@ -495,8 +506,9 @@ public class NamenodeWebHdfsMethods {
{
final URI uri = redirectURI(namenode, ugi, delegation, username,
doAsUser, fullpath, op.getValue(), -1L, blockSize.getValue(conf),
- exclDatanodes.getValue(), permission, overwrite, bufferSize,
- replication, blockSize, createParent, createFlagParam);
+ exclDatanodes.getValue(), permission, unmaskedPermission,
+ overwrite, bufferSize, replication, blockSize, createParent,
+ createFlagParam);
if(!noredirectParam.getValue()) {
return Response.temporaryRedirect(uri)
.type(MediaType.APPLICATION_OCTET_STREAM).build();
@@ -507,8 +519,11 @@ public class NamenodeWebHdfsMethods {
}
case MKDIRS:
{
- final boolean b = np.mkdirs(fullpath,
- permission.getDirFsPermission(), true);
+ FsPermission masked = unmaskedPermission.getValue() == null ?
+ permission.getDirFsPermission() :
+ FsCreateModes.create(permission.getDirFsPermission(),
+ unmaskedPermission.getDirFsPermission());
+ final boolean b = np.mkdirs(fullpath, masked, true);
final String js = JsonUtil.toJsonString("boolean", b);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 4eca80c..d427f88 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -457,7 +457,19 @@
</description>
</property>
-<property>
+ <property>
+ <name>dfs.namenode.posix.acl.inheritance.enabled</name>
+ <value>false</value>
+ <description>
+ Set to true to enable POSIX style ACL inheritance. When it is enabled
+ and the create request comes from a compatible client, the NameNode
+ will apply default ACLs from the parent directory to the create mode
+ and ignore the client umask. If no default ACL found, it will apply the
+ client umask.
+ </description>
+ </property>
+
+ <property>
<name>dfs.namenode.lazypersist.file.scrub.interval.sec</name>
<value>300</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
index ba027ac..7fd9b00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
@@ -333,4 +333,10 @@ Configuration Parameters
default, ACLs are disabled. When ACLs are disabled, the NameNode rejects
all attempts to set an ACL.
+* `dfs.namenode.posix.acl.inheritance.enabled`
+ Set to true to enable POSIX style ACL inheritance. Disabled by default.
+ When it is enabled and the create request comes from a compatible client,
+ the NameNode will apply default ACLs from the parent directory to
+ the create mode and ignore the client umask. If no default ACL is found,
+ it will apply the client umask.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java
index 061d328..75111bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java
@@ -32,11 +32,15 @@ public class TestAclCLI extends CLITestHelperDFS {
private String namenode = null;
private String username = null;
+ protected void initConf() {
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ }
+
@Before
@Override
public void setUp() throws Exception {
super.setUp();
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ initConf();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
fs = cluster.getFileSystem();
namenode = conf.get(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "file:///");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLIWithPosixAclInheritance.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLIWithPosixAclInheritance.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLIWithPosixAclInheritance.java
new file mode 100644
index 0000000..ec31766
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLIWithPosixAclInheritance.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cli;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY;
+
+import org.junit.Test;
+
+/**
+ * Test ACL CLI with POSIX ACL inheritance enabled.
+ */
+public class TestAclCLIWithPosixAclInheritance extends TestAclCLI {
+
+ @Override
+ protected void initConf() {
+ super.initConf();
+ conf.setBoolean(DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY, true);
+ }
+
+ @Override
+ protected String getTestFile() {
+ return "testAclCLIWithPosixAclInheritance.xml";
+ }
+
+ @Test
+ @Override
+ public void testAll() {
+ super.testAll();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
index 216147a..71614f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY;
import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.*;
import static org.apache.hadoop.fs.permission.AclEntryScope.*;
import static org.apache.hadoop.fs.permission.AclEntryType.*;
@@ -889,6 +890,46 @@ public abstract class FSAclBaseTest {
}
@Test
+ public void testUMaskDefaultAclNewFile() throws Exception {
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, GROUP, READ_WRITE),
+ aclEntry(DEFAULT, USER, "foo", ALL));
+ fs.setAcl(path, aclSpec);
+
+ String oldUMask = fs.getConf().get(FS_PERMISSIONS_UMASK_KEY);
+ fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, "027");
+
+ FSDirectory fsDirectory = cluster.getNamesystem().getFSDirectory();
+ boolean oldEnabled = fsDirectory.isPosixAclInheritanceEnabled();
+
+ try {
+ fsDirectory.setPosixAclInheritanceEnabled(false);
+ Path filePath = new Path(path, "file1");
+ fs.create(filePath).close();
+ AclStatus s = fs.getAclStatus(filePath);
+ AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[]{
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_WRITE)}, returned);
+ assertPermission(filePath, (short) 010640);
+
+ fsDirectory.setPosixAclInheritanceEnabled(true);
+ Path file2Path = new Path(path, "file2");
+ fs.create(file2Path).close();
+ AclStatus s2 = fs.getAclStatus(file2Path);
+ AclEntry[] returned2 = s2.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[]{
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_WRITE)}, returned2);
+ assertPermission(file2Path, (short) 010660);
+ } finally {
+ fsDirectory.setPosixAclInheritanceEnabled(oldEnabled);
+ fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, oldUMask);
+ }
+ }
+
+ @Test
public void testOnlyAccessAclNewFile() throws Exception {
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -943,6 +984,56 @@ public abstract class FSAclBaseTest {
}
@Test
+ public void testUMaskDefaultAclNewDir() throws Exception {
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, GROUP, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL));
+ fs.setAcl(path, aclSpec);
+
+ String oldUMask = fs.getConf().get(FS_PERMISSIONS_UMASK_KEY);
+ fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, "027");
+
+ FSDirectory fsDirectory = cluster.getNamesystem().getFSDirectory();
+ boolean oldEnabled = fsDirectory.isPosixAclInheritanceEnabled();
+
+ try {
+ fsDirectory.setPosixAclInheritanceEnabled(false);
+ Path dirPath = new Path(path, "dir1");
+ fs.mkdirs(dirPath);
+ AclStatus s = fs.getAclStatus(dirPath);
+ AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[]{
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, ALL),
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL),
+ aclEntry(DEFAULT, GROUP, ALL),
+ aclEntry(DEFAULT, MASK, ALL),
+ aclEntry(DEFAULT, OTHER, NONE)}, returned);
+ assertPermission(dirPath, (short) 010750);
+
+ fsDirectory.setPosixAclInheritanceEnabled(true);
+ Path dir2Path = new Path(path, "dir2");
+ fs.mkdirs(dir2Path);
+ AclStatus s2 = fs.getAclStatus(dir2Path);
+ AclEntry[] returned2 = s2.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[]{
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, ALL),
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL),
+ aclEntry(DEFAULT, GROUP, ALL),
+ aclEntry(DEFAULT, MASK, ALL),
+ aclEntry(DEFAULT, OTHER, NONE)}, returned2);
+ assertPermission(dir2Path, (short) 010770);
+ } finally {
+ fsDirectory.setPosixAclInheritanceEnabled(oldEnabled);
+ fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, oldUMask);
+ }
+ }
+
+ @Test
public void testOnlyAccessAclNewDir() throws Exception {
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
List<AclEntry> aclSpec = Lists.newArrayList(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
index eec5c98..0eb7132 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
@@ -127,7 +127,7 @@ public class TestGetBlockLocations {
MOCK_INODE_ID, FILE_NAME.getBytes(StandardCharsets.UTF_8),
perm, 1, 1, new BlockInfo[] {}, (short) 1,
DFS_BLOCK_SIZE_DEFAULT);
- fsn.getFSDirectory().addINode(iip, file);
+ fsn.getFSDirectory().addINode(iip, file, null);
return fsn;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
index ad3a5a1..91eec78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
@@ -1570,7 +1570,9 @@ public class TestRenameWithSnapshots {
FSDirectory fsdir2 = Mockito.spy(fsdir);
Mockito.doThrow(new NSQuotaExceededException("fake exception")).when(fsdir2)
.addLastINode((INodesInPath) Mockito.anyObject(),
- (INode) Mockito.anyObject(), Mockito.anyBoolean());
+ (INode) Mockito.anyObject(),
+ (FsPermission) Mockito.anyObject(),
+ Mockito.anyBoolean());
Whitebox.setInternalState(fsn, "dir", fsdir2);
// rename /test/dir1/foo to /test/dir2/subdir2/foo.
// FSDirectory#verifyQuota4Rename will pass since the remaining quota is 2.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/22] hadoop git commit: HADOOP-13483. File create should throw
error rather than overwrite directories. Contributed by Genmao Yu.
Posted by sh...@apache.org.
HADOOP-13483. File create should throw error rather than overwrite directories. Contributed by Genmao Yu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd2d97ad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd2d97ad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd2d97ad
Branch: refs/heads/HADOOP-12756
Commit: bd2d97adeea55bf2c7e4ab475bcc90f3a14e751a
Parents: 4d84c81
Author: Mingfei <mi...@intel.com>
Authored: Tue Aug 23 16:18:34 2016 +0800
Committer: Mingfei <mi...@intel.com>
Committed: Wed Sep 7 11:15:47 2016 +0800
----------------------------------------------------------------------
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 25 +++++++++++++++++---
.../oss/contract/TestOSSContractCreate.java | 6 -----
2 files changed, 22 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd2d97ad/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 6923b95..99a60db 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -63,6 +63,8 @@ import com.aliyun.oss.model.ObjectMetadata;
import com.aliyun.oss.model.PartETag;
import com.aliyun.oss.model.UploadPartCopyRequest;
import com.aliyun.oss.model.UploadPartCopyResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Implementation of {@link FileSystem} for <a href="https://oss.aliyun.com">
@@ -70,6 +72,8 @@ import com.aliyun.oss.model.UploadPartCopyResult;
*/
public class AliyunOSSFileSystem extends FileSystem {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AliyunOSSFileSystem.class);
private URI uri;
private Path workingDir;
private OSSClient ossClient;
@@ -101,9 +105,24 @@ public class AliyunOSSFileSystem extends FileSystem {
boolean overwrite, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
String key = pathToKey(path);
+ FileStatus status = null;
- if (!overwrite && exists(path)) {
- throw new FileAlreadyExistsException(path + " already exists");
+ try {
+ // get the status or throw a FNFE
+ status = getFileStatus(path);
+
+ // if the thread reaches here, there is something at the path
+ if (status.isDirectory()) {
+ // path references a directory
+ throw new FileAlreadyExistsException(path + " is a directory");
+ }
+ if (!overwrite) {
+ // path references a file and overwrite is disabled
+ throw new FileAlreadyExistsException(path + " already exists");
+ }
+ LOG.debug("Overwriting file {}", path);
+ } catch (FileNotFoundException e) {
+ // this means the file is not found
}
return new FSDataOutputStream(new AliyunOSSOutputStream(getConf(),
@@ -540,7 +559,7 @@ public class AliyunOSSFileSystem extends FileSystem {
/**
* Used to create an empty file that represents an empty directory.
*
- * @param bucketName the bucket this directory belongs to
+ * @param bucket the bucket this directory belongs to
* @param objectName directory path
* @return true if directory successfully created
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd2d97ad/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java
index cc5a2d1..ce927a9 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.fs.aliyun.oss.contract;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
/**
* OSS contract creating tests.
@@ -33,9 +32,4 @@ public class TestOSSContractCreate extends AbstractContractCreateTest {
return new OSSContract(conf);
}
- @Override
- public void testOverwriteEmptyDirectory() throws Throwable {
- ContractTestUtils.skip(
- "blobstores can't distinguish empty directories from files");
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/22] hadoop git commit: HADOOP-13498. The number of multi-part
upload part should not bigger than 10000. Contributed by Genmao Yu.
Posted by sh...@apache.org.
HADOOP-13498. The number of multi-part upload part should not bigger than 10000. Contributed by Genmao Yu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdb77110
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdb77110
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdb77110
Branch: refs/heads/HADOOP-12756
Commit: cdb77110e77b70ed0c1125b2a6a422a8c7c28ec7
Parents: 6bb741b
Author: Mingfei <mi...@intel.com>
Authored: Wed Aug 24 10:09:37 2016 +0800
Committer: Mingfei <mi...@intel.com>
Committed: Wed Sep 7 11:15:48 2016 +0800
----------------------------------------------------------------------
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 9 ++++----
.../fs/aliyun/oss/AliyunOSSOutputStream.java | 23 ++++++++++----------
.../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 15 +++++++++++++
.../apache/hadoop/fs/aliyun/oss/Constants.java | 4 +++-
.../fs/aliyun/oss/TestOSSOutputStream.java | 19 ++++++++++++++++
5 files changed, 53 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb77110/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 99a60db..afe7242 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -71,7 +71,6 @@ import org.slf4j.LoggerFactory;
* Aliyun OSS</a>, used to access OSS blob system in a filesystem style.
*/
public class AliyunOSSFileSystem extends FileSystem {
-
private static final Logger LOG =
LoggerFactory.getLogger(AliyunOSSFileSystem.class);
private URI uri;
@@ -560,18 +559,18 @@ public class AliyunOSSFileSystem extends FileSystem {
* Used to create an empty file that represents an empty directory.
*
* @param bucket the bucket this directory belongs to
- * @param objectName directory path
+ * @param key directory path
* @return true if directory successfully created
* @throws IOException
*/
- private boolean mkdir(final String bucket, final String objectName)
+ private boolean mkdir(final String bucket, final String key)
throws IOException {
- String dirName = objectName;
+ String dirName = key;
ObjectMetadata dirMeta = new ObjectMetadata();
byte[] buffer = new byte[0];
ByteArrayInputStream in = new ByteArrayInputStream(buffer);
dirMeta.setContentLength(0);
- if (!objectName.endsWith("/")) {
+ if (!key.endsWith("/")) {
dirName += "/";
}
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb77110/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
index 654b81d..1e16df9 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
@@ -84,6 +84,9 @@ public class AliyunOSSOutputStream extends OutputStream {
partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
MULTIPART_UPLOAD_SIZE_DEFAULT);
+ if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) {
+ partSize = MIN_MULTIPART_UPLOAD_PART_SIZE;
+ }
partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
@@ -151,6 +154,12 @@ public class AliyunOSSOutputStream extends OutputStream {
private void multipartUploadObject() throws IOException {
File object = tmpFile.getAbsoluteFile();
long dataLen = object.length();
+ long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize);
+ int partNum = (int)(dataLen / realPartSize);
+ if (dataLen % realPartSize != 0) {
+ partNum += 1;
+ }
+
InitiateMultipartUploadRequest initiateMultipartUploadRequest =
new InitiateMultipartUploadRequest(bucketName, key);
ObjectMetadata meta = new ObjectMetadata();
@@ -161,14 +170,6 @@ public class AliyunOSSOutputStream extends OutputStream {
initiateMultipartUploadRequest.setObjectMetadata(meta);
InitiateMultipartUploadResult initiateMultipartUploadResult =
ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
- int partNum = (int)(dataLen / partSize);
- if (dataLen % partSize != 0) {
- partNum += 1;
- }
- if (partNum > MULTIPART_UPLOAD_PART_NUM_LIMIT) {
- throw new IOException("Number of parts " + partNum + " should not be " +
- "bigger than limit " + MULTIPART_UPLOAD_PART_NUM_LIMIT);
- }
List<PartETag> partETags = new ArrayList<PartETag>();
String uploadId = initiateMultipartUploadResult.getUploadId();
@@ -177,10 +178,10 @@ public class AliyunOSSOutputStream extends OutputStream {
// TODO: Optimize this, avoid opening the object multiple times
FileInputStream fis = new FileInputStream(object);
try {
- long skipBytes = partSize * i;
+ long skipBytes = realPartSize * i;
AliyunOSSUtils.skipFully(fis, skipBytes);
- long size = (partSize < dataLen - skipBytes) ?
- partSize : dataLen - skipBytes;
+ long size = (realPartSize < dataLen - skipBytes) ?
+ realPartSize : dataLen - skipBytes;
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(bucketName);
uploadPartRequest.setKey(key);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb77110/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
index 9acde00..d54dd9c 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
@@ -28,6 +28,8 @@ import java.util.Objects;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.fs.aliyun.oss.Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT;
+
/**
* Utility methods for Aliyun OSS code.
*/
@@ -172,4 +174,17 @@ final public class AliyunOSSUtils {
"to EOF.");
}
}
+
+ /**
+ * Calculate a proper size of multipart piece. If <code>minPartSize</code>
+ * is too small, the number of multipart pieces may exceed the limit of
+ * {@link Constants#MULTIPART_UPLOAD_PART_NUM_LIMIT}.
+ * @param contentLength the size of file.
+ * @param minPartSize the minimum size of multipart piece.
+ * @return a revisional size of multipart piece.
+ */
+ public static long calculatePartSize(long contentLength, long minPartSize) {
+ long tmpPartSize = contentLength / MULTIPART_UPLOAD_PART_NUM_LIMIT + 1;
+ return Math.max(minPartSize, tmpPartSize);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb77110/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
index 0bc6d57..9902275 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -79,7 +79,7 @@ public final class Constants {
"fs.oss.multipart.upload.size";
public static final long MULTIPART_UPLOAD_SIZE_DEFAULT = 10 * 1024 * 1024;
- public static final int MULTIPART_UPLOAD_PART_NUM_LIMIT = 1000;
+ public static final int MULTIPART_UPLOAD_PART_NUM_LIMIT = 10000;
// Minimum size in bytes before we start a multipart uploads or copy
public static final String MIN_MULTIPART_UPLOAD_THRESHOLD_KEY =
@@ -108,4 +108,6 @@ public final class Constants {
public static final int FS_OSS_BLOCK_SIZE_DEFAULT = 64 * 1024 * 1024;
public static final String FS_OSS = "oss";
+ public static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 100 * 1024L;
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb77110/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java
index 3951529..b33ab99 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java
@@ -68,4 +68,23 @@ public class TestOSSOutputStream {
public void testMultiPartUpload() throws IOException {
ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 * 1024);
}
+
+ @Test
+ public void testMultiPartUploadLimit() throws IOException {
+ long partSize1 = AliyunOSSUtils.calculatePartSize(10 * 1024, 100 * 1024);
+ assert(10 * 1024 / partSize1 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+
+ long partSize2 = AliyunOSSUtils.calculatePartSize(200 * 1024, 100 * 1024);
+ assert(200 * 1024 / partSize2 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+
+ long partSize3 = AliyunOSSUtils.calculatePartSize(10000 * 100 * 1024,
+ 100 * 1024);
+ assert(10000 * 100 * 1024 / partSize3
+ < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+
+ long partSize4 = AliyunOSSUtils.calculatePartSize(10001 * 100 * 1024,
+ 100 * 1024);
+ assert(10001 * 100 * 1024 / partSize4
+ < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/22] hadoop git commit: HADOOP-13447. Refactor S3AFileSystem to
support introduction of separate metadata repository and tests. Contributed
by Chris Nauroth.
Posted by sh...@apache.org.
HADOOP-13447. Refactor S3AFileSystem to support introduction of separate metadata repository and tests. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d152557c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d152557c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d152557c
Branch: refs/heads/HADOOP-12756
Commit: d152557cf7f4d2288524c222fcbaf152bdc038b0
Parents: 39d1b1d
Author: Chris Nauroth <cn...@apache.org>
Authored: Tue Sep 6 09:36:21 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Tue Sep 6 09:36:21 2016 -0700
----------------------------------------------------------------------
hadoop-tools/hadoop-aws/pom.xml | 5 +
.../org/apache/hadoop/fs/s3a/Constants.java | 29 ++-
.../hadoop/fs/s3a/S3AFastOutputStream.java | 6 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 192 ++-------------
.../apache/hadoop/fs/s3a/S3AInputStream.java | 6 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 40 ++++
.../apache/hadoop/fs/s3a/S3ClientFactory.java | 232 +++++++++++++++++++
.../hadoop/fs/s3native/S3xLoginHelper.java | 3 +
.../hadoop/fs/s3a/AbstractS3AMockTest.java | 70 ++++++
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 10 +-
.../hadoop/fs/s3a/MockS3ClientFactory.java | 40 ++++
.../hadoop/fs/s3a/TestS3AGetFileStatus.java | 126 ++++++++++
12 files changed, 567 insertions(+), 192 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 82a8581..13dcdf1 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -296,6 +296,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 6b8f9f5..64fd8e5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -53,7 +53,8 @@ public final class Constants {
public static final int DEFAULT_MAXIMUM_CONNECTIONS = 15;
// connect to s3 over ssl?
- public static final String SECURE_CONNECTIONS = "fs.s3a.connection.ssl.enabled";
+ public static final String SECURE_CONNECTIONS =
+ "fs.s3a.connection.ssl.enabled";
public static final boolean DEFAULT_SECURE_CONNECTIONS = true;
//use a custom endpoint?
@@ -75,7 +76,8 @@ public final class Constants {
public static final int DEFAULT_MAX_ERROR_RETRIES = 20;
// seconds until we give up trying to establish a connection to s3
- public static final String ESTABLISH_TIMEOUT = "fs.s3a.connection.establish.timeout";
+ public static final String ESTABLISH_TIMEOUT =
+ "fs.s3a.connection.establish.timeout";
public static final int DEFAULT_ESTABLISH_TIMEOUT = 50000;
// seconds until we give up on a connection to s3
@@ -111,11 +113,13 @@ public final class Constants {
public static final long DEFAULT_MULTIPART_SIZE = 104857600; // 100 MB
// minimum size in bytes before we start a multipart uploads or copy
- public static final String MIN_MULTIPART_THRESHOLD = "fs.s3a.multipart.threshold";
+ public static final String MIN_MULTIPART_THRESHOLD =
+ "fs.s3a.multipart.threshold";
public static final long DEFAULT_MIN_MULTIPART_THRESHOLD = Integer.MAX_VALUE;
//enable multiobject-delete calls?
- public static final String ENABLE_MULTI_DELETE = "fs.s3a.multiobjectdelete.enable";
+ public static final String ENABLE_MULTI_DELETE =
+ "fs.s3a.multiobjectdelete.enable";
// comma separated list of directories
public static final String BUFFER_DIR = "fs.s3a.buffer.dir";
@@ -134,11 +138,13 @@ public final class Constants {
public static final String DEFAULT_CANNED_ACL = "";
// should we try to purge old multipart uploads when starting up
- public static final String PURGE_EXISTING_MULTIPART = "fs.s3a.multipart.purge";
+ public static final String PURGE_EXISTING_MULTIPART =
+ "fs.s3a.multipart.purge";
public static final boolean DEFAULT_PURGE_EXISTING_MULTIPART = false;
// purge any multipart uploads older than this number of seconds
- public static final String PURGE_EXISTING_MULTIPART_AGE = "fs.s3a.multipart.purge.age";
+ public static final String PURGE_EXISTING_MULTIPART_AGE =
+ "fs.s3a.multipart.purge.age";
public static final long DEFAULT_PURGE_EXISTING_MULTIPART_AGE = 14400;
// s3 server-side encryption
@@ -198,4 +204,15 @@ public final class Constants {
*/
@InterfaceStability.Unstable
public static final String INPUT_FADV_RANDOM = "random";
+
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public static final String S3_CLIENT_FACTORY_IMPL =
+ "fs.s3a.s3.client.factory.impl";
+
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public static final Class<? extends S3ClientFactory>
+ DEFAULT_S3_CLIENT_FACTORY_IMPL =
+ S3ClientFactory.DefaultS3ClientFactory.class;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
index 5509d36..c25d0fb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a;
import com.amazonaws.AmazonClientException;
import com.amazonaws.event.ProgressEvent;
import com.amazonaws.event.ProgressListener;
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
@@ -71,7 +71,7 @@ public class S3AFastOutputStream extends OutputStream {
private static final Logger LOG = S3AFileSystem.LOG;
private final String key;
private final String bucket;
- private final AmazonS3Client client;
+ private final AmazonS3 client;
private final int partSize;
private final int multiPartThreshold;
private final S3AFileSystem fs;
@@ -102,7 +102,7 @@ public class S3AFastOutputStream extends OutputStream {
* @param threadPoolExecutor thread factory
* @throws IOException on any problem
*/
- public S3AFastOutputStream(AmazonS3Client client,
+ public S3AFastOutputStream(AmazonS3 client,
S3AFileSystem fs,
String bucket,
String key,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index f8c65da..6a6c26b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -34,11 +34,7 @@ import java.util.concurrent.TimeUnit;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.S3ClientOptions;
+import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
@@ -57,7 +53,6 @@ import com.amazonaws.services.s3.transfer.Upload;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.event.ProgressEvent;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -78,7 +73,7 @@ import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.util.ReflectionUtils;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.Listing.ACCEPT_ALL;
@@ -110,7 +105,7 @@ public class S3AFileSystem extends FileSystem {
public static final int DEFAULT_BLOCKSIZE = 32 * 1024 * 1024;
private URI uri;
private Path workingDir;
- private AmazonS3Client s3;
+ private AmazonS3 s3;
private String bucket;
private int maxKeys;
private Listing listing;
@@ -147,37 +142,11 @@ public class S3AFileSystem extends FileSystem {
bucket = name.getHost();
- AWSCredentialsProvider credentials =
- createAWSCredentialProviderSet(name, conf, uri);
-
- ClientConfiguration awsConf = new ClientConfiguration();
- awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS,
- DEFAULT_MAXIMUM_CONNECTIONS, 1));
- boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
- DEFAULT_SECURE_CONNECTIONS);
- awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
- awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES,
- DEFAULT_MAX_ERROR_RETRIES, 0));
- awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT,
- DEFAULT_ESTABLISH_TIMEOUT, 0));
- awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT,
- DEFAULT_SOCKET_TIMEOUT, 0));
- int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER,
- DEFAULT_SOCKET_SEND_BUFFER, 2048);
- int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER,
- DEFAULT_SOCKET_RECV_BUFFER, 2048);
- awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
- String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
- if (!signerOverride.isEmpty()) {
- LOG.debug("Signer override = {}", signerOverride);
- awsConf.setSignerOverride(signerOverride);
- }
-
- initProxySupport(conf, awsConf, secureConnections);
-
- initUserAgent(conf, awsConf);
-
- initAmazonS3Client(conf, credentials, awsConf);
+ Class<? extends S3ClientFactory> s3ClientFactoryClass = conf.getClass(
+ S3_CLIENT_FACTORY_IMPL, DEFAULT_S3_CLIENT_FACTORY_IMPL,
+ S3ClientFactory.class);
+ s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf)
+ .createS3Client(name, uri);
maxKeys = intOption(conf, MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS, 1);
listing = new Listing(this);
@@ -266,50 +235,6 @@ public class S3AFileSystem extends FileSystem {
}
}
- void initProxySupport(Configuration conf, ClientConfiguration awsConf,
- boolean secureConnections) throws IllegalArgumentException {
- String proxyHost = conf.getTrimmed(PROXY_HOST, "");
- int proxyPort = conf.getInt(PROXY_PORT, -1);
- if (!proxyHost.isEmpty()) {
- awsConf.setProxyHost(proxyHost);
- if (proxyPort >= 0) {
- awsConf.setProxyPort(proxyPort);
- } else {
- if (secureConnections) {
- LOG.warn("Proxy host set without port. Using HTTPS default 443");
- awsConf.setProxyPort(443);
- } else {
- LOG.warn("Proxy host set without port. Using HTTP default 80");
- awsConf.setProxyPort(80);
- }
- }
- String proxyUsername = conf.getTrimmed(PROXY_USERNAME);
- String proxyPassword = conf.getTrimmed(PROXY_PASSWORD);
- if ((proxyUsername == null) != (proxyPassword == null)) {
- String msg = "Proxy error: " + PROXY_USERNAME + " or " +
- PROXY_PASSWORD + " set without the other.";
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- awsConf.setProxyUsername(proxyUsername);
- awsConf.setProxyPassword(proxyPassword);
- awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
- awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
- if (LOG.isDebugEnabled()) {
- LOG.debug("Using proxy server {}:{} as user {} with password {} on " +
- "domain {} as workstation {}", awsConf.getProxyHost(),
- awsConf.getProxyPort(),
- String.valueOf(awsConf.getProxyUsername()),
- awsConf.getProxyPassword(), awsConf.getProxyDomain(),
- awsConf.getProxyWorkstation());
- }
- } else if (proxyPort >= 0) {
- String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- }
-
/**
* Get S3A Instrumentation. For test purposes.
* @return this instance's instrumentation.
@@ -318,53 +243,9 @@ public class S3AFileSystem extends FileSystem {
return instrumentation;
}
- /**
- * Initializes the User-Agent header to send in HTTP requests to the S3
- * back-end. We always include the Hadoop version number. The user also may
- * set an optional custom prefix to put in front of the Hadoop version number.
- * The AWS SDK interally appends its own information, which seems to include
- * the AWS SDK version, OS and JVM version.
- *
- * @param conf Hadoop configuration
- * @param awsConf AWS SDK configuration
- */
- private void initUserAgent(Configuration conf, ClientConfiguration awsConf) {
- String userAgent = "Hadoop " + VersionInfo.getVersion();
- String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, "");
- if (!userAgentPrefix.isEmpty()) {
- userAgent = userAgentPrefix + ", " + userAgent;
- }
- LOG.debug("Using User-Agent: {}", userAgent);
- awsConf.setUserAgent(userAgent);
- }
-
- private void initAmazonS3Client(Configuration conf,
- AWSCredentialsProvider credentials, ClientConfiguration awsConf)
- throws IllegalArgumentException {
- s3 = new AmazonS3Client(credentials, awsConf);
- String endPoint = conf.getTrimmed(ENDPOINT, "");
- if (!endPoint.isEmpty()) {
- try {
- s3.setEndpoint(endPoint);
- } catch (IllegalArgumentException e) {
- String msg = "Incorrect endpoint: " + e.getMessage();
- LOG.error(msg);
- throw new IllegalArgumentException(msg, e);
- }
- }
- enablePathStyleAccessIfRequired(conf);
- }
-
- private void enablePathStyleAccessIfRequired(Configuration conf) {
- final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
- if (pathStyleAccess) {
- LOG.debug("Enabling path style access!");
- s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
- }
- }
-
private void initTransferManager() {
- TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
+ TransferManagerConfiguration transferConfiguration =
+ new TransferManagerConfiguration();
transferConfiguration.setMinimumUploadPartSize(partSize);
transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
transferConfiguration.setMultipartCopyPartSize(partSize);
@@ -435,7 +316,7 @@ public class S3AFileSystem extends FileSystem {
* @return AmazonS3Client
*/
@VisibleForTesting
- AmazonS3Client getAmazonS3Client() {
+ AmazonS3 getAmazonS3Client() {
return s3;
}
@@ -459,10 +340,6 @@ public class S3AFileSystem extends FileSystem {
this.inputPolicy = inputPolicy;
}
- public S3AFileSystem() {
- super();
- }
-
/**
* Turns a path (relative or otherwise) into an S3 key.
*
@@ -782,8 +659,10 @@ public class S3AFileSystem extends FileSystem {
while (true) {
for (S3ObjectSummary summary : objects.getObjectSummaries()) {
- keysToDelete.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
- String newDstKey = dstKey + summary.getKey().substring(srcKey.length());
+ keysToDelete.add(
+ new DeleteObjectsRequest.KeyVersion(summary.getKey()));
+ String newDstKey =
+ dstKey + summary.getKey().substring(srcKey.length());
copyFile(summary.getKey(), newDstKey, summary.getSize());
if (keysToDelete.size() == MAX_ENTRIES_TO_DELETE) {
@@ -1388,7 +1267,8 @@ public class S3AFileSystem extends FileSystem {
LOG.debug("Found file (with /): fake directory");
return new S3AFileStatus(true, true, path);
} else {
- LOG.warn("Found file (with /): real file? should not happen: {}", key);
+ LOG.warn("Found file (with /): real file? should not happen: {}",
+ key);
return new S3AFileStatus(meta.getContentLength(),
dateToLong(meta.getLastModified()),
@@ -1985,42 +1865,4 @@ public class S3AFileSystem extends FileSystem {
getFileBlockLocations(status, 0, status.getLen())
: null);
}
-
- /**
- * Get a integer option >= the minimum allowed value.
- * @param conf configuration
- * @param key key to look up
- * @param defVal default value
- * @param min minimum value
- * @return the value
- * @throws IllegalArgumentException if the value is below the minimum
- */
- static int intOption(Configuration conf, String key, int defVal, int min) {
- int v = conf.getInt(key, defVal);
- Preconditions.checkArgument(v >= min,
- String.format("Value of %s: %d is below the minimum value %d",
- key, v, min));
- return v;
- }
-
- /**
- * Get a long option >= the minimum allowed value.
- * @param conf configuration
- * @param key key to look up
- * @param defVal default value
- * @param min minimum value
- * @return the value
- * @throws IllegalArgumentException if the value is below the minimum
- */
- static long longOption(Configuration conf,
- String key,
- long defVal,
- long min) {
- long v = conf.getLong(key, defVal);
- Preconditions.checkArgument(v >= min,
- String.format("Value of %s: %d is below the minimum value %d",
- key, v, min));
- return v;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index ccb9726..dd6cdd7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.fs.s3a;
import com.amazonaws.AmazonClientException;
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.google.common.base.Preconditions;
@@ -71,7 +71,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
private volatile boolean closed;
private S3ObjectInputStream wrappedStream;
private final FileSystem.Statistics stats;
- private final AmazonS3Client client;
+ private final AmazonS3 client;
private final String bucket;
private final String key;
private final long contentLength;
@@ -101,7 +101,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
public S3AInputStream(String bucket,
String key,
long contentLength,
- AmazonS3Client client,
+ AmazonS3 client,
FileSystem.Statistics stats,
S3AInstrumentation instrumentation,
long readahead,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 699676d..a5e8e7a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -25,6 +25,9 @@ import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+import com.google.common.base.Preconditions;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -403,4 +406,41 @@ public final class S3AUtils {
builder.append("size=").append(summary.getSize());
return builder.toString();
}
+
+ /**
+ * Get a integer option >= the minimum allowed value.
+ * @param conf configuration
+ * @param key key to look up
+ * @param defVal default value
+ * @param min minimum value
+ * @return the value
+ * @throws IllegalArgumentException if the value is below the minimum
+ */
+ static int intOption(Configuration conf, String key, int defVal, int min) {
+ int v = conf.getInt(key, defVal);
+ Preconditions.checkArgument(v >= min,
+ String.format("Value of %s: %d is below the minimum value %d",
+ key, v, min));
+ return v;
+ }
+
+ /**
+ * Get a long option >= the minimum allowed value.
+ * @param conf configuration
+ * @param key key to look up
+ * @param defVal default value
+ * @param min minimum value
+ * @return the value
+ * @throws IllegalArgumentException if the value is below the minimum
+ */
+ static long longOption(Configuration conf,
+ String key,
+ long defVal,
+ long min) {
+ long v = conf.getLong(key, defVal);
+ Preconditions.checkArgument(v >= min,
+ String.format("Value of %s: %d is below the minimum value %d",
+ key, v, min));
+ return v;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
new file mode 100644
index 0000000..0a4dd02
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+
+import java.io.IOException;
+import java.net.URI;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.S3ClientOptions;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.VersionInfo;
+
+import org.slf4j.Logger;
+
+/**
+ * Factory for creation of S3 client instances to be used by {@link S3Store}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+interface S3ClientFactory {
+
+ /**
+ * Creates a new {@link AmazonS3} client. This method accepts the S3A file
+ * system URI both in raw input form and validated form as separate arguments,
+ * because both values may be useful in logging.
+ *
+ * @param name raw input S3A file system URI
+ * @param uri validated form of S3A file system URI
+ * @return S3 client
+ * @throws IOException IO problem
+ */
+ AmazonS3 createS3Client(URI name, URI uri) throws IOException;
+
+ /**
+ * The default factory implementation, which calls the AWS SDK to configure
+ * and create an {@link AmazonS3Client} that communicates with the S3 service.
+ */
+ static class DefaultS3ClientFactory extends Configured
+ implements S3ClientFactory {
+
+ private static final Logger LOG = S3AFileSystem.LOG;
+
+ @Override
+ public AmazonS3 createS3Client(URI name, URI uri) throws IOException {
+ Configuration conf = getConf();
+ AWSCredentialsProvider credentials =
+ createAWSCredentialProviderSet(name, conf, uri);
+ ClientConfiguration awsConf = new ClientConfiguration();
+ initConnectionSettings(conf, awsConf);
+ initProxySupport(conf, awsConf);
+ initUserAgent(conf, awsConf);
+ return createAmazonS3Client(conf, credentials, awsConf);
+ }
+
+ /**
+ * Initializes all AWS SDK settings related to connection management.
+ *
+ * @param conf Hadoop configuration
+ * @param awsConf AWS SDK configuration
+ */
+ private static void initConnectionSettings(Configuration conf,
+ ClientConfiguration awsConf) {
+ awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS,
+ DEFAULT_MAXIMUM_CONNECTIONS, 1));
+ boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
+ DEFAULT_SECURE_CONNECTIONS);
+ awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
+ awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES,
+ DEFAULT_MAX_ERROR_RETRIES, 0));
+ awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT,
+ DEFAULT_ESTABLISH_TIMEOUT, 0));
+ awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT,
+ DEFAULT_SOCKET_TIMEOUT, 0));
+ int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER,
+ DEFAULT_SOCKET_SEND_BUFFER, 2048);
+ int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER,
+ DEFAULT_SOCKET_RECV_BUFFER, 2048);
+ awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
+ String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
+ if (!signerOverride.isEmpty()) {
+ LOG.debug("Signer override = {}", signerOverride);
+ awsConf.setSignerOverride(signerOverride);
+ }
+ }
+
+ /**
+ * Initializes AWS SDK proxy support if configured.
+ *
+ * @param conf Hadoop configuration
+ * @param awsConf AWS SDK configuration
+ * @throws IllegalArgumentException if misconfigured
+ */
+ private static void initProxySupport(Configuration conf,
+ ClientConfiguration awsConf) throws IllegalArgumentException {
+ String proxyHost = conf.getTrimmed(PROXY_HOST, "");
+ int proxyPort = conf.getInt(PROXY_PORT, -1);
+ if (!proxyHost.isEmpty()) {
+ awsConf.setProxyHost(proxyHost);
+ if (proxyPort >= 0) {
+ awsConf.setProxyPort(proxyPort);
+ } else {
+ if (conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS)) {
+ LOG.warn("Proxy host set without port. Using HTTPS default 443");
+ awsConf.setProxyPort(443);
+ } else {
+ LOG.warn("Proxy host set without port. Using HTTP default 80");
+ awsConf.setProxyPort(80);
+ }
+ }
+ String proxyUsername = conf.getTrimmed(PROXY_USERNAME);
+ String proxyPassword = conf.getTrimmed(PROXY_PASSWORD);
+ if ((proxyUsername == null) != (proxyPassword == null)) {
+ String msg = "Proxy error: " + PROXY_USERNAME + " or " +
+ PROXY_PASSWORD + " set without the other.";
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+ awsConf.setProxyUsername(proxyUsername);
+ awsConf.setProxyPassword(proxyPassword);
+ awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
+ awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using proxy server {}:{} as user {} with password {} on " +
+ "domain {} as workstation {}", awsConf.getProxyHost(),
+ awsConf.getProxyPort(),
+ String.valueOf(awsConf.getProxyUsername()),
+ awsConf.getProxyPassword(), awsConf.getProxyDomain(),
+ awsConf.getProxyWorkstation());
+ }
+ } else if (proxyPort >= 0) {
+ String msg =
+ "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+ }
+
+ /**
+ * Initializes the User-Agent header to send in HTTP requests to the S3
+ * back-end. We always include the Hadoop version number. The user also
+ * may set an optional custom prefix to put in front of the Hadoop version
+ * number. The AWS SDK interally appends its own information, which seems
+ * to include the AWS SDK version, OS and JVM version.
+ *
+ * @param conf Hadoop configuration
+ * @param awsConf AWS SDK configuration
+ */
+ private static void initUserAgent(Configuration conf,
+ ClientConfiguration awsConf) {
+ String userAgent = "Hadoop " + VersionInfo.getVersion();
+ String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, "");
+ if (!userAgentPrefix.isEmpty()) {
+ userAgent = userAgentPrefix + ", " + userAgent;
+ }
+ LOG.debug("Using User-Agent: {}", userAgent);
+ awsConf.setUserAgent(userAgent);
+ }
+
+ /**
+ * Creates an {@link AmazonS3Client} from the established configuration.
+ *
+ * @param conf Hadoop configuration
+ * @param credentials AWS credentials
+ * @param awsConf AWS SDK configuration
+ * @return S3 client
+ * @throws IllegalArgumentException if misconfigured
+ */
+ private static AmazonS3 createAmazonS3Client(Configuration conf,
+ AWSCredentialsProvider credentials, ClientConfiguration awsConf)
+ throws IllegalArgumentException {
+ AmazonS3 s3 = new AmazonS3Client(credentials, awsConf);
+ String endPoint = conf.getTrimmed(ENDPOINT, "");
+ if (!endPoint.isEmpty()) {
+ try {
+ s3.setEndpoint(endPoint);
+ } catch (IllegalArgumentException e) {
+ String msg = "Incorrect endpoint: " + e.getMessage();
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg, e);
+ }
+ }
+ enablePathStyleAccessIfRequired(s3, conf);
+ return s3;
+ }
+
+ /**
+ * Enables path-style access to S3 buckets if configured. By default, the
+ * behavior is to use virtual hosted-style access with URIs of the form
+ * http://bucketname.s3.amazonaws.com. Enabling path-style access and a
+ * region-specific endpoint switches the behavior to use URIs of the form
+ * http://s3-eu-west-1.amazonaws.com/bucketname.
+ *
+ * @param s3 S3 client
+ * @param conf Hadoop configuration
+ */
+ private static void enablePathStyleAccessIfRequired(AmazonS3 s3,
+ Configuration conf) {
+ final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
+ if (pathStyleAccess) {
+ LOG.debug("Enabling path style access!");
+ s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
index bc8c2e6..97ece37 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
@@ -132,6 +132,9 @@ public final class S3xLoginHelper {
*
* This strips out login information.
*
+ * @param uri the URI to canonicalize
+ * @param defaultPort default port to use in canonicalized URI if the input
+ * URI has no port and this value is greater than 0
* @return a new, canonicalized URI.
*/
public static URI canonicalizeUri(URI uri, int defaultPort) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
new file mode 100644
index 0000000..6734947
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.services.s3.AmazonS3;
+
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Abstract base class for S3A unit tests using a mock S3 client.
+ */
+public abstract class AbstractS3AMockTest {
+
+ protected static final String BUCKET = "mock-bucket";
+ protected static final AmazonServiceException NOT_FOUND;
+ static {
+ NOT_FOUND = new AmazonServiceException("Not Found");
+ NOT_FOUND.setStatusCode(404);
+ }
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ protected S3AFileSystem fs;
+ protected AmazonS3 s3;
+
+ @Before
+ public void setup() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class,
+ S3ClientFactory.class);
+ fs = new S3AFileSystem();
+ URI uri = URI.create(FS_S3A + "://" + BUCKET);
+ fs.initialize(uri, conf);
+ s3 = fs.getAmazonS3Client();
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (fs != null) {
+ fs.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index 4e99339..fca8e49 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.fs.s3a;
import com.amazonaws.ClientConfiguration;
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.S3ClientOptions;
import org.apache.commons.lang.StringUtils;
@@ -96,7 +96,7 @@ public class ITestS3AConfiguration {
} else {
conf.set(Constants.ENDPOINT, endpoint);
fs = S3ATestUtils.createTestFileSystem(conf);
- AmazonS3Client s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3Client();
String endPointRegion = "";
// Differentiate handling of "s3-" and "s3." based endpoint identifiers
String[] endpointParts = StringUtils.split(endpoint, '.');
@@ -364,7 +364,7 @@ public class ITestS3AConfiguration {
try {
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3Client();
assertNotNull(s3);
S3ClientOptions clientOptions = getField(s3, S3ClientOptions.class,
"clientOptions");
@@ -388,7 +388,7 @@ public class ITestS3AConfiguration {
conf = new Configuration();
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3Client();
assertNotNull(s3);
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
@@ -401,7 +401,7 @@ public class ITestS3AConfiguration {
conf.set(Constants.USER_AGENT_PREFIX, "MyApp");
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3Client();
assertNotNull(s3);
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
new file mode 100644
index 0000000..41f04ee
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.mockito.Mockito.*;
+
+import java.net.URI;
+
+import com.amazonaws.services.s3.AmazonS3;
+
+/**
+ * An {@link S3ClientFactory} that returns Mockito mocks of the {@link AmazonS3}
+ * interface suitable for unit testing.
+ */
+public class MockS3ClientFactory implements S3ClientFactory {
+
+ @Override
+ public AmazonS3 createS3Client(URI name, URI uri) {
+ String bucket = name.getHost();
+ AmazonS3 s3 = mock(AmazonS3.class);
+ when(s3.doesBucketExist(bucket)).thenReturn(true);
+ return s3;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
new file mode 100644
index 0000000..f9e9c6b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.*;
+import static org.mockito.Mockito.*;
+
+import java.io.FileNotFoundException;
+import java.util.Collections;
+import java.util.Date;
+
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import org.junit.Test;
+
+/**
+ * S3A tests for getFileStatus using mock S3 client.
+ */
+public class TestS3AGetFileStatus extends AbstractS3AMockTest {
+
+ @Test
+ public void testFile() throws Exception {
+ Path path = new Path("/file");
+ String key = path.toUri().getPath().substring(1);
+ ObjectMetadata meta = new ObjectMetadata();
+ meta.setContentLength(1L);
+ meta.setLastModified(new Date(2L));
+ when(s3.getObjectMetadata(BUCKET, key)).thenReturn(meta);
+ FileStatus stat = fs.getFileStatus(path);
+ assertNotNull(stat);
+ assertEquals(fs.makeQualified(path), stat.getPath());
+ assertTrue(stat.isFile());
+ assertEquals(meta.getContentLength(), stat.getLen());
+ assertEquals(meta.getLastModified().getTime(), stat.getModificationTime());
+ }
+
+ @Test
+ public void testFakeDirectory() throws Exception {
+ Path path = new Path("/dir");
+ String key = path.toUri().getPath().substring(1);
+ when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ ObjectMetadata meta = new ObjectMetadata();
+ meta.setContentLength(0L);
+ when(s3.getObjectMetadata(BUCKET, key + "/")).thenReturn(meta);
+ FileStatus stat = fs.getFileStatus(path);
+ assertNotNull(stat);
+ assertEquals(fs.makeQualified(path), stat.getPath());
+ assertTrue(stat.isDirectory());
+ }
+
+ @Test
+ public void testImplicitDirectory() throws Exception {
+ Path path = new Path("/dir");
+ String key = path.toUri().getPath().substring(1);
+ when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(BUCKET, key + "/")).thenThrow(NOT_FOUND);
+ ObjectListing objects = mock(ObjectListing.class);
+ when(objects.getCommonPrefixes()).thenReturn(
+ Collections.singletonList("dir/"));
+ when(objects.getObjectSummaries()).thenReturn(
+ Collections.<S3ObjectSummary>emptyList());
+ when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
+ FileStatus stat = fs.getFileStatus(path);
+ assertNotNull(stat);
+ assertEquals(fs.makeQualified(path), stat.getPath());
+ assertTrue(stat.isDirectory());
+ }
+
+ @Test
+ public void testRoot() throws Exception {
+ Path path = new Path("/");
+ String key = path.toUri().getPath().substring(1);
+ when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(BUCKET, key + "/")).thenThrow(NOT_FOUND);
+ ObjectListing objects = mock(ObjectListing.class);
+ when(objects.getCommonPrefixes()).thenReturn(
+ Collections.<String>emptyList());
+ when(objects.getObjectSummaries()).thenReturn(
+ Collections.<S3ObjectSummary>emptyList());
+ when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
+ FileStatus stat = fs.getFileStatus(path);
+ assertNotNull(stat);
+ assertEquals(fs.makeQualified(path), stat.getPath());
+ assertTrue(stat.isDirectory());
+ assertTrue(stat.getPath().isRoot());
+ }
+
+ @Test
+ public void testNotFound() throws Exception {
+ Path path = new Path("/dir");
+ String key = path.toUri().getPath().substring(1);
+ when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(BUCKET, key + "/")).thenThrow(NOT_FOUND);
+ ObjectListing objects = mock(ObjectListing.class);
+ when(objects.getCommonPrefixes()).thenReturn(
+ Collections.<String>emptyList());
+ when(objects.getObjectSummaries()).thenReturn(
+ Collections.<S3ObjectSummary>emptyList());
+ when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
+ exception.expect(FileNotFoundException.class);
+ fs.getFileStatus(path);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org