You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/06/26 05:32:39 UTC
[01/50] [abbrv] hadoop git commit: HDFS-13673. TestNameNodeMetrics
fails on Windows. Contributed by Zuoming Zhang.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-12943 292ccdce8 -> 831097335
HDFS-13673. TestNameNodeMetrics fails on Windows. Contributed by Zuoming Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43d994e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43d994e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43d994e4
Branch: refs/heads/HDFS-12943
Commit: 43d994e4a6dfd1c24eafb909d6f8a0663b20769a
Parents: b272b71
Author: Inigo Goiri <in...@apache.org>
Authored: Fri Jun 15 10:33:28 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Fri Jun 15 10:33:28 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d994e4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index e34deea..05cf2ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
@@ -271,7 +272,8 @@ public class TestNameNodeMetrics {
File dataDir = new File(fsVolume.getBaseURI());
long capacity = fsVolume.getCapacity();
volumeReferences.close();
- DataNodeTestUtils.injectDataDirFailure(dataDir);
+ File storageDir = new File(dataDir, Storage.STORAGE_DIR_CURRENT);
+ DataNodeTestUtils.injectDataDirFailure(storageDir);
DataNodeTestUtils.waitForDiskError(dn, fsVolume);
DataNodeTestUtils.triggerHeartbeat(dn);
BlockManagerTestUtil.checkHeartbeat(bm);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDDS-177. Create a releasable
ozonefs artifact Contributed by Marton, Elek.
Posted by xk...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
deleted file mode 100644
index 3156eb2..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-import java.io.IOException;
-
-/**
- * Ozone contract test for ROOT directory operations.
- */
-public class ITestOzoneContractRootDir extends
- AbstractContractRootDirectoryTest {
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
deleted file mode 100644
index c4bc0ff..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests covering file seek.
- */
-public class ITestOzoneContractSeek extends AbstractContractSeekTest {
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
deleted file mode 100644
index 8417e46..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.junit.Assert;
-
-import java.io.IOException;
-
-/**
- * The contract of Ozone: only enabled if the test bucket is provided.
- */
-class OzoneContract extends AbstractFSContract {
-
- private static MiniOzoneCluster cluster;
- private static StorageHandler storageHandler;
- private static final String CONTRACT_XML = "contract/ozone.xml";
-
- OzoneContract(Configuration conf) {
- super(conf);
- //insert the base features
- addConfResource(CONTRACT_XML);
- }
-
- @Override
- public String getScheme() {
- return OzoneConsts.OZONE_URI_SCHEME;
- }
-
- @Override
- public Path getTestPath() {
- Path path = new Path("/test");
- return path;
- }
-
- public static void createCluster() throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.addResource(CONTRACT_XML);
-
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
- try {
- cluster.waitForClusterToBeReady();
- } catch (Exception e) {
- throw new IOException(e);
- }
- storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
- }
-
- private void copyClusterConfigs(String configKey) {
- getConf().set(configKey, cluster.getConf().get(configKey));
- }
-
- @Override
- public FileSystem getTestFileSystem() throws IOException {
- //assumes cluster is not null
- Assert.assertNotNull("cluster not created", cluster);
-
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-
-
- UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- try {
- storageHandler.createVolume(volumeArgs);
- storageHandler.createBucket(bucketArgs);
- } catch (OzoneException e) {
- throw new IOException(e.getMessage());
- }
-
- String uri = String.format("%s://%s.%s/",
- OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
- getConf().set("fs.defaultFS", uri);
- copyClusterConfigs(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY);
- copyClusterConfigs(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
- return FileSystem.get(getConf());
- }
-
- public static void destroyCluster() throws IOException {
- if (cluster != null) {
- cluster.shutdown();
- cluster = null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml b/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml
deleted file mode 100644
index fe2075c..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml
+++ /dev/null
@@ -1,113 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration>
- <!--
- Ozone is a blobstore, with very different behavior than a classic filesystem.
- -->
-
- <property>
- <name>fs.contract.test.root-tests-enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.test.random-seek-count</name>
- <value>10</value>
- </property>
-
- <property>
- <name>fs.contract.is-blobstore</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.create-visibility-delayed</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.is-case-sensitive</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.rename-returns-false-if-source-missing</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.rename-remove-dest-if-empty-dir</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-append</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-atomic-directory-delete</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-atomic-rename</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-block-locality</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-concat</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-getfilestatus</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-seek</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-seek-on-closed-file</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.rejects-seek-past-eof</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-strict-exceptions</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-unix-permissions</name>
- <value>false</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties b/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties
deleted file mode 100644
index 3bf1619..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# log4j configuration used during build and unit tests
-
-log4j.rootLogger=INFO,stdout
-log4j.threshold=ALL
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
-
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-
-# for debugging low level Ozone operations, uncomment this line
-# log4j.logger.org.apache.hadoop.ozone=DEBUG
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 21cc7ce..42ce94c 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -192,20 +192,5 @@
</plugins>
</build>
</profile>
-
- <profile>
- <id>hdds</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-filesystem</artifactId>
- <scope>compile</scope>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index f421e58..dca59d3 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -67,15 +67,4 @@
</plugin>
</plugins>
</build>
- <profiles>
- <profile>
- <id>hdds</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <modules>
- <module>hadoop-ozone</module>
- </modules>
- </profile>
- </profiles>
</project>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HADOOP-15504. Upgrade Maven Core
and Maven Wagon dependencies.
Posted by xk...@apache.org.
HADOOP-15504. Upgrade Maven Core and Maven Wagon dependencies.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1da3b556
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1da3b556
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1da3b556
Branch: refs/heads/HDFS-12943
Commit: 1da3b556591fffaae0751c4f7fceda34d2314fda
Parents: 308a159
Author: Sean Mackrory <ma...@apache.org>
Authored: Tue May 1 08:56:10 2018 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Fri Jun 15 16:06:26 2018 -0600
----------------------------------------------------------------------
hadoop-maven-plugins/pom.xml | 38 +++++++++++++++-----
.../maven/plugin/resourcegz/ResourceGzMojo.java | 2 +-
pom.xml | 2 +-
3 files changed, 31 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1da3b556/hadoop-maven-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index b31d158..d6b18b4 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -26,7 +26,7 @@
<packaging>maven-plugin</packaging>
<name>Apache Hadoop Maven Plugins</name>
<properties>
- <maven.dependency.version>3.0</maven.dependency.version>
+ <maven.dependency.version>3.0.5</maven.dependency.version>
<maven.plugin-tools.version>3.5.1</maven.plugin-tools.version>
</properties>
<dependencies>
@@ -45,6 +45,14 @@
<artifactId>maven-plugin-annotations</artifactId>
<version>${maven.plugin-tools.version}</version>
<scope>provided</scope>
+ <exclusions>
+ <!-- excluding dependency that conflicts with other maven
+ dependencies -->
+ <exclusion>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-artifact</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>commons-io</groupId>
@@ -60,16 +68,28 @@
<version>${maven-shade-plugin.version}</version>
<scope>provided</scope>
<exclusions>
- <!-- shade conflicts with itself and our other maven internals
- by including a combination of 3.x and 2.x
- -->
+ <!-- shade conflicts with itself and other maven internals so we simply
+ exclude shade's (older) copies of the dependencies -->
+ <exclusion>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-artifact</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-compat</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-model</artifactId>
+ </exclusion>
<exclusion>
- <groupId>org.apache.maven.shared</groupId>
- <artifactId>maven-dependency-tree</artifactId>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-plugin-api</artifactId>
</exclusion>
- <!-- again, shade conflicts with itself on the transitives of this
- dependency
- -->
<exclusion>
<groupId>org.vafer</groupId>
<artifactId>jdependency</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1da3b556/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/resourcegz/ResourceGzMojo.java
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/resourcegz/ResourceGzMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/resourcegz/ResourceGzMojo.java
index e7ab663..5bf84c2 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/resourcegz/ResourceGzMojo.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/resourcegz/ResourceGzMojo.java
@@ -13,7 +13,7 @@
*/
package org.apache.hadoop.maven.plugin.resourcegz;
-import com.google.inject.internal.util.Lists;
+import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1da3b556/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3695be0..a250e64 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,7 +106,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<maven-remote-resources-plugin.version>1.5</maven-remote-resources-plugin.version>
<maven-resources-plugin.version>3.0.1</maven-resources-plugin.version>
<apache-rat-plugin.version>0.12</apache-rat-plugin.version>
- <wagon-ssh.version>1.0</wagon-ssh.version>
+ <wagon-ssh.version>2.4</wagon-ssh.version>
<maven-clover2-plugin.version>3.3.0</maven-clover2-plugin.version>
<maven-bundle-plugin.version>2.5.0</maven-bundle-plugin.version>
<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HDFS-13676. TestEditLogRace fails
on Windows. Contributed by Zuoming Zhang.
Posted by xk...@apache.org.
HDFS-13676. TestEditLogRace fails on Windows. Contributed by Zuoming Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eebeb603
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eebeb603
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eebeb603
Branch: refs/heads/HDFS-12943
Commit: eebeb6033fd09791fcbff626f128a98e393f0a88
Parents: 43d994e
Author: Inigo Goiri <in...@apache.org>
Authored: Fri Jun 15 13:07:54 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Fri Jun 15 13:07:54 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/namenode/TestEditLogRace.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eebeb603/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
index 46010e0..10f571c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
@@ -84,6 +84,8 @@ public class TestEditLogRace {
TestEditLogRace.useAsyncEditLog = useAsyncEditLog;
}
+ private static final String NAME_DIR = MiniDFSCluster.getBaseDirectory() + "name-0-1";
+
private static final Log LOG = LogFactory.getLog(TestEditLogRace.class);
// This test creates NUM_THREADS threads and each thread continuously writes
@@ -363,8 +365,8 @@ public class TestEditLogRace {
useAsyncEditLog);
FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
- //conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, NAME_DIR);
- //conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, NAME_DIR);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, NAME_DIR);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, NAME_DIR);
conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false);
return conf;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-8391. Investigate
AllocationFileLoaderService.reloadListener locking issue. Contributed by
Szilard Nemeth.
Posted by xk...@apache.org.
YARN-8391. Investigate AllocationFileLoaderService.reloadListener locking issue. Contributed by Szilard Nemeth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a9e9695
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a9e9695
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a9e9695
Branch: refs/heads/HDFS-12943
Commit: 9a9e969570f23b627f9571819f388916d8fd7ec9
Parents: 4939ffe
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Wed Jun 20 10:29:12 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Wed Jun 20 10:59:33 2018 -0700
----------------------------------------------------------------------
.../scheduler/fair/AllocationFileLoaderService.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a9e9695/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index 56cc887..3300948 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -114,7 +114,9 @@ public class AllocationFileLoaderService extends AbstractService {
reloadThread = new Thread(() -> {
while (running) {
try {
- reloadListener.onCheck();
+ synchronized (this) {
+ reloadListener.onCheck();
+ }
long time = clock.getTime();
long lastModified =
fs.getFileStatus(allocFile).getModificationTime();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HADOOP-13186. Multipart Uploader
API. Contributed by Ewan Higgs
Posted by xk...@apache.org.
HADOOP-13186. Multipart Uploader API. Contributed by Ewan Higgs
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/980031bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/980031bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/980031bb
Branch: refs/heads/HDFS-12943
Commit: 980031bb043dd026a6bf42b0e71d304ac89294a5
Parents: 3905fdb
Author: Chris Douglas <cd...@apache.org>
Authored: Sun Jun 17 11:54:26 2018 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Sun Jun 17 11:54:26 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/BBPartHandle.java | 58 +++++++
.../org/apache/hadoop/fs/BBUploadHandle.java | 57 +++++++
.../hadoop/fs/FileSystemMultipartUploader.java | 132 ++++++++++++++++
.../hadoop/fs/LocalFileSystemPathHandle.java | 100 +++++++++++++
.../org/apache/hadoop/fs/MultipartUploader.java | 90 +++++++++++
.../hadoop/fs/MultipartUploaderFactory.java | 65 ++++++++
.../java/org/apache/hadoop/fs/PartHandle.java | 45 ++++++
.../apache/hadoop/fs/RawLocalFileSystem.java | 61 +++++++-
.../UnsupportedMultipartUploaderException.java | 41 +++++
.../java/org/apache/hadoop/fs/UploadHandle.java | 47 ++++++
.../hadoop-common/src/main/proto/FSProtos.proto | 8 +
...rg.apache.hadoop.fs.MultipartUploaderFactory | 16 ++
.../fs/AbstractSystemMultipartUploaderTest.java | 143 ++++++++++++++++++
.../TestLocalFileSystemMultipartUploader.java | 65 ++++++++
.../AbstractContractPathHandleTest.java | 6 +
.../TestRawlocalContractPathHandle.java | 40 +++++
.../src/test/resources/contract/rawlocal.xml | 5 +
.../hdfs/DFSMultipartUploaderFactory.java | 40 +++++
...rg.apache.hadoop.fs.MultipartUploaderFactory | 16 ++
.../hadoop/fs/TestHDFSMultipartUploader.java | 76 ++++++++++
.../hadoop/fs/s3a/S3AMultipartUploader.java | 150 +++++++++++++++++++
...rg.apache.hadoop.fs.MultipartUploaderFactory | 15 ++
.../org.apache.hadoop.fs.MultipartUploader | 16 ++
23 files changed, 1290 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BBPartHandle.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BBPartHandle.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BBPartHandle.java
new file mode 100644
index 0000000..e1336b8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BBPartHandle.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * Byte array backed part handle.
+ */
+public final class BBPartHandle implements PartHandle {
+
+ private static final long serialVersionUID = 0x23ce3eb1;
+
+ private final byte[] bytes;
+
+ private BBPartHandle(ByteBuffer byteBuffer){
+ this.bytes = byteBuffer.array();
+ }
+
+ public static PartHandle from(ByteBuffer byteBuffer) {
+ return new BBPartHandle(byteBuffer);
+ }
+
+ @Override
+ public ByteBuffer bytes() {
+ return ByteBuffer.wrap(bytes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(bytes);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof PartHandle)) {
+ return false;
+
+ }
+ PartHandle o = (PartHandle) other;
+ return bytes().equals(o.bytes());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BBUploadHandle.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BBUploadHandle.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BBUploadHandle.java
new file mode 100644
index 0000000..6430c14
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BBUploadHandle.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * Byte array backed upload handle.
+ */
+public final class BBUploadHandle implements UploadHandle {
+
+ private static final long serialVersionUID = 0x69d5509b;
+
+ private final byte[] bytes;
+
+ private BBUploadHandle(ByteBuffer byteBuffer){
+ this.bytes = byteBuffer.array();
+ }
+
+ public static UploadHandle from(ByteBuffer byteBuffer) {
+ return new BBUploadHandle(byteBuffer);
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(bytes);
+ }
+
+ @Override
+ public ByteBuffer bytes() {
+ return ByteBuffer.wrap(bytes);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof UploadHandle)) {
+ return false;
+ }
+ UploadHandle o = (UploadHandle) other;
+ return bytes().equals(o.bytes());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
new file mode 100644
index 0000000..b57ff3d
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsPermission;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * A MultipartUploader that uses the basic FileSystem commands.
+ * This is done in three stages:
+ * Init - create a temp _multipart directory.
+ * PutPart - copying the individual parts of the file to the temp directory.
+ * Complete - use {@link FileSystem#concat} to merge the files; and then delete
+ * the temp directory.
+ */
+public class FileSystemMultipartUploader extends MultipartUploader {
+
+ private final FileSystem fs;
+
+ public FileSystemMultipartUploader(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ @Override
+ public UploadHandle initialize(Path filePath) throws IOException {
+ Path collectorPath = createCollectorPath(filePath);
+ fs.mkdirs(collectorPath, FsPermission.getDirDefault());
+
+ ByteBuffer byteBuffer = ByteBuffer.wrap(
+ collectorPath.toString().getBytes(Charsets.UTF_8));
+ return BBUploadHandle.from(byteBuffer);
+ }
+
+ @Override
+ public PartHandle putPart(Path filePath, InputStream inputStream,
+ int partNumber, UploadHandle uploadId, long lengthInBytes)
+ throws IOException {
+
+ byte[] uploadIdByteArray = uploadId.toByteArray();
+ Path collectorPath = new Path(new String(uploadIdByteArray, 0,
+ uploadIdByteArray.length, Charsets.UTF_8));
+ Path partPath =
+ Path.mergePaths(collectorPath, Path.mergePaths(new Path(Path.SEPARATOR),
+ new Path(Integer.toString(partNumber) + ".part")));
+ FSDataOutputStreamBuilder outputStream = fs.createFile(partPath);
+ FSDataOutputStream fsDataOutputStream = outputStream.build();
+ IOUtils.copy(inputStream, fsDataOutputStream, 4096);
+ fsDataOutputStream.close();
+ return BBPartHandle.from(ByteBuffer.wrap(
+ partPath.toString().getBytes(Charsets.UTF_8)));
+ }
+
+ private Path createCollectorPath(Path filePath) {
+ return Path.mergePaths(filePath.getParent(),
+ Path.mergePaths(new Path(filePath.getName().split("\\.")[0]),
+ Path.mergePaths(new Path("_multipart"),
+ new Path(Path.SEPARATOR))));
+ }
+
+ @Override
+ @SuppressWarnings("deprecation") // rename w/ OVERWRITE
+ public PathHandle complete(Path filePath,
+ List<Pair<Integer, PartHandle>> handles, UploadHandle multipartUploadId)
+ throws IOException {
+ handles.sort(Comparator.comparing(Pair::getKey));
+ List<Path> partHandles = handles
+ .stream()
+ .map(pair -> {
+ byte[] byteArray = pair.getValue().toByteArray();
+ return new Path(new String(byteArray, 0, byteArray.length,
+ Charsets.UTF_8));
+ })
+ .collect(Collectors.toList());
+
+ Path collectorPath = createCollectorPath(filePath);
+ Path filePathInsideCollector = Path.mergePaths(collectorPath,
+ new Path(Path.SEPARATOR + filePath.getName()));
+ fs.create(filePathInsideCollector).close();
+ fs.concat(filePathInsideCollector,
+ partHandles.toArray(new Path[handles.size()]));
+ fs.rename(filePathInsideCollector, filePath, Options.Rename.OVERWRITE);
+ fs.delete(collectorPath, true);
+ FileStatus status = fs.getFileStatus(filePath);
+ return fs.getPathHandle(status);
+ }
+
+ @Override
+ public void abort(Path filePath, UploadHandle uploadId) throws IOException {
+ byte[] uploadIdByteArray = uploadId.toByteArray();
+ Path collectorPath = new Path(new String(uploadIdByteArray, 0,
+ uploadIdByteArray.length, Charsets.UTF_8));
+ fs.delete(collectorPath, true);
+ }
+
+ /**
+ * Factory for creating MultipartUploaderFactory objects for file://
+ * filesystems.
+ */
+ public static class Factory extends MultipartUploaderFactory {
+ protected MultipartUploader createMultipartUploader(FileSystem fs,
+ Configuration conf) {
+ if (fs.getScheme().equals("file")) {
+ return new FileSystemMultipartUploader(fs);
+ }
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystemPathHandle.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystemPathHandle.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystemPathHandle.java
new file mode 100644
index 0000000..a6b37b3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystemPathHandle.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import com.google.protobuf.ByteString;
+import org.apache.hadoop.fs.FSProtos.LocalFileSystemPathHandleProto;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Opaque handle to an entity in a FileSystem.
+ */
+public class LocalFileSystemPathHandle implements PathHandle {
+
+ private final String path;
+ private final Long mtime;
+
+ public LocalFileSystemPathHandle(String path, Optional<Long> mtime) {
+ this.path = path;
+ this.mtime = mtime.orElse(null);
+ }
+
+ public LocalFileSystemPathHandle(ByteBuffer bytes) throws IOException {
+ if (null == bytes) {
+ throw new IOException("Missing PathHandle");
+ }
+ LocalFileSystemPathHandleProto p =
+ LocalFileSystemPathHandleProto.parseFrom(ByteString.copyFrom(bytes));
+ path = p.hasPath() ? p.getPath() : null;
+ mtime = p.hasMtime() ? p.getMtime() : null;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void verify(FileStatus stat) throws InvalidPathHandleException {
+ if (null == stat) {
+ throw new InvalidPathHandleException("Could not resolve handle");
+ }
+ if (mtime != null && mtime != stat.getModificationTime()) {
+ throw new InvalidPathHandleException("Content changed");
+ }
+ }
+
+ @Override
+ public ByteBuffer bytes() {
+ LocalFileSystemPathHandleProto.Builder b =
+ LocalFileSystemPathHandleProto.newBuilder();
+ b.setPath(path);
+ if (mtime != null) {
+ b.setMtime(mtime);
+ }
+ return b.build().toByteString().asReadOnlyByteBuffer();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ LocalFileSystemPathHandle that = (LocalFileSystemPathHandle) o;
+ return Objects.equals(path, that.path) &&
+ Objects.equals(mtime, that.mtime);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(path, mtime);
+ }
+
+ @Override
+ public String toString() {
+ return "LocalFileSystemPathHandle{" +
+ "path='" + path + '\'' +
+ ", mtime=" + mtime +
+ '}';
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
new file mode 100644
index 0000000..24a9216
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import org.apache.commons.lang3.tuple.Pair;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * MultipartUploader is an interface for copying files multipart and across
+ * multiple nodes. Users should:
+ * 1. Initialize an upload
+ * 2. Upload parts in any order
+ * 3. Complete the upload in order to have it materialize in the destination FS.
+ *
+ * Implementers should make sure that the complete function should make sure
+ * that 'complete' will reorder parts if the destination FS doesn't already
+ * do it for them.
+ */
+public abstract class MultipartUploader {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(MultipartUploader.class);
+
+ /**
+ * Initialize a multipart upload.
+ * @param filePath Target path for upload.
+ * @return unique identifier associating part uploads.
+ * @throws IOException
+ */
+ public abstract UploadHandle initialize(Path filePath) throws IOException;
+
+ /**
+ * Put part as part of a multipart upload. It should be possible to have
+ * parts uploaded in any order (or in parallel).
+ * @param filePath Target path for upload (same as {@link #initialize(Path)}).
+ * @param inputStream Data for this part.
+ * @param partNumber Index of the part relative to others.
+ * @param uploadId Identifier from {@link #initialize(Path)}.
+ * @param lengthInBytes Target length to read from the stream.
+ * @return unique PartHandle identifier for the uploaded part.
+ * @throws IOException
+ */
+ public abstract PartHandle putPart(Path filePath, InputStream inputStream,
+ int partNumber, UploadHandle uploadId, long lengthInBytes)
+ throws IOException;
+
+ /**
+ * Complete a multipart upload.
+ * @param filePath Target path for upload (same as {@link #initialize(Path)}.
+ * @param handles Identifiers with associated part numbers from
+ * {@link #putPart(Path, InputStream, int, UploadHandle, long)}.
+ * Depending on the backend, the list order may be significant.
+ * @param multipartUploadId Identifier from {@link #initialize(Path)}.
+ * @return unique PathHandle identifier for the uploaded file.
+ * @throws IOException
+ */
+ public abstract PathHandle complete(Path filePath,
+ List<Pair<Integer, PartHandle>> handles, UploadHandle multipartUploadId)
+ throws IOException;
+
+ /**
+ * Aborts a multipart upload.
+ * @param filePath Target path for upload (same as {@link #initialize(Path)}.
+ * @param multipartuploadId Identifier from {@link #initialize(Path)}.
+ * @throws IOException
+ */
+ public abstract void abort(Path filePath, UploadHandle multipartuploadId)
+ throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
new file mode 100644
index 0000000..b0fa798
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.ServiceLoader;
+
+/**
+ * {@link ServiceLoader}-driven uploader API for storage services supporting
+ * multipart uploads.
+ */
+public abstract class MultipartUploaderFactory {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(MultipartUploaderFactory.class);
+
+ /**
+ * Multipart Uploaders listed as services.
+ */
+ private static ServiceLoader<MultipartUploaderFactory> serviceLoader =
+ ServiceLoader.load(MultipartUploaderFactory.class,
+ MultipartUploaderFactory.class.getClassLoader());
+
+ // Iterate through the serviceLoader to avoid lazy loading.
+ // Lazy loading would require synchronization in concurrent use cases.
+ static {
+ Iterator<MultipartUploaderFactory> iterServices = serviceLoader.iterator();
+ while (iterServices.hasNext()) {
+ iterServices.next();
+ }
+ }
+
+ public static MultipartUploader get(FileSystem fs, Configuration conf)
+ throws IOException {
+ MultipartUploader mpu = null;
+ for (MultipartUploaderFactory factory : serviceLoader) {
+ mpu = factory.createMultipartUploader(fs, conf);
+ if (mpu != null) {
+ break;
+ }
+ }
+ return mpu;
+ }
+
+ protected abstract MultipartUploader createMultipartUploader(FileSystem fs,
+ Configuration conf) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PartHandle.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PartHandle.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PartHandle.java
new file mode 100644
index 0000000..df70b74
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PartHandle.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+
+/**
+ * Opaque, serializable reference to an part id for multipart uploads.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface PartHandle extends Serializable {
+ /**
+ * @return Serialized from in bytes.
+ */
+ default byte[] toByteArray() {
+ ByteBuffer bb = bytes();
+ byte[] ret = new byte[bb.remaining()];
+ bb.get(ret);
+ return ret;
+ }
+
+ ByteBuffer bytes();
+
+ @Override
+ boolean equals(Object other);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index c0f8199..bd003ae 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -40,6 +40,7 @@ import java.nio.file.attribute.BasicFileAttributeView;
import java.nio.file.attribute.FileTime;
import java.util.Arrays;
import java.util.EnumSet;
+import java.util.Optional;
import java.util.StringTokenizer;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -212,7 +213,19 @@ public class RawLocalFileSystem extends FileSystem {
return new FSDataInputStream(new BufferedFSInputStream(
new LocalFSFileInputStream(f), bufferSize));
}
-
+
+ @Override
+ public FSDataInputStream open(PathHandle fd, int bufferSize)
+ throws IOException {
+ if (!(fd instanceof LocalFileSystemPathHandle)) {
+ fd = new LocalFileSystemPathHandle(fd.bytes());
+ }
+ LocalFileSystemPathHandle id = (LocalFileSystemPathHandle) fd;
+ id.verify(getFileStatus(new Path(id.getPath())));
+ return new FSDataInputStream(new BufferedFSInputStream(
+ new LocalFSFileInputStream(new Path(id.getPath())), bufferSize));
+ }
+
/*********************************************************
* For create()'s FSOutputStream.
*********************************************************/
@@ -246,7 +259,7 @@ public class RawLocalFileSystem extends FileSystem {
}
}
}
-
+
/*
* Just forward to the fos
*/
@@ -351,6 +364,18 @@ public class RawLocalFileSystem extends FileSystem {
}
@Override
+ public void concat(final Path trg, final Path [] psrcs) throws IOException {
+ final int bufferSize = 4096;
+ try(FSDataOutputStream out = create(trg)) {
+ for (Path src : psrcs) {
+ try(FSDataInputStream in = open(src)) {
+ IOUtils.copyBytes(in, out, bufferSize, false);
+ }
+ }
+ }
+ }
+
+ @Override
public boolean rename(Path src, Path dst) throws IOException {
// Attempt rename using Java API.
File srcFile = pathToFile(src);
@@ -863,6 +888,38 @@ public class RawLocalFileSystem extends FileSystem {
}
}
+ /**
+ * Hook to implement support for {@link PathHandle} operations.
+ * @param stat Referent in the target FileSystem
+ * @param opts Constraints that determine the validity of the
+ * {@link PathHandle} reference.
+ */
+ protected PathHandle createPathHandle(FileStatus stat,
+ Options.HandleOpt... opts) {
+ if (stat.isDirectory() || stat.isSymlink()) {
+ throw new IllegalArgumentException("PathHandle only available for files");
+ }
+ String authority = stat.getPath().toUri().getAuthority();
+ if (authority != null && !authority.equals("file://")) {
+ throw new IllegalArgumentException("Wrong FileSystem: " + stat.getPath());
+ }
+ Options.HandleOpt.Data data =
+ Options.HandleOpt.getOpt(Options.HandleOpt.Data.class, opts)
+ .orElse(Options.HandleOpt.changed(false));
+ Options.HandleOpt.Location loc =
+ Options.HandleOpt.getOpt(Options.HandleOpt.Location.class, opts)
+ .orElse(Options.HandleOpt.moved(false));
+ if (loc.allowChange()) {
+ throw new UnsupportedOperationException("Tracking file movement in " +
+ "basic FileSystem is not supported");
+ }
+ final Path p = stat.getPath();
+ final Optional<Long> mtime = !data.allowChange()
+ ? Optional.of(stat.getModificationTime())
+ : Optional.empty();
+ return new LocalFileSystemPathHandle(p.toString(), mtime);
+ }
+
@Override
public boolean supportsSymlinks() {
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/UnsupportedMultipartUploaderException.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/UnsupportedMultipartUploaderException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/UnsupportedMultipartUploaderException.java
new file mode 100644
index 0000000..5606a80
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/UnsupportedMultipartUploaderException.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * MultipartUploader for a given file system name/scheme is not supported.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class UnsupportedMultipartUploaderException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Constructs exception with the specified detail message.
+ *
+ * @param message exception message.
+ */
+ public UnsupportedMultipartUploaderException(final String message) {
+ super(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/UploadHandle.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/UploadHandle.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/UploadHandle.java
new file mode 100644
index 0000000..143b4d1
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/UploadHandle.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+
+/**
+ * Opaque, serializable reference to an uploadId for multipart uploads.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface UploadHandle extends Serializable {
+
+ /**
+ * @return Serialized from in bytes.
+ */
+ default byte[] toByteArray() {
+ ByteBuffer bb = bytes();
+ byte[] ret = new byte[bb.remaining()];
+ bb.get(ret);
+ return ret;
+ }
+
+ ByteBuffer bytes();
+
+ @Override
+ boolean equals(Object other);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
index 5b8c45d..c3b768a 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
@@ -68,3 +68,11 @@ message FileStatusProto {
optional bytes ec_data = 17;
optional uint32 flags = 18 [default = 0];
}
+
+/**
+ * Placeholder type for consistent basic FileSystem operations.
+ */
+message LocalFileSystemPathHandleProto {
+ optional uint64 mtime = 1;
+ optional string path = 2;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.MultipartUploaderFactory
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.MultipartUploaderFactory b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.MultipartUploaderFactory
new file mode 100644
index 0000000..f0054fe
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.MultipartUploaderFactory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.fs.FileSystemMultipartUploader$Factory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/AbstractSystemMultipartUploaderTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/AbstractSystemMultipartUploaderTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/AbstractSystemMultipartUploaderTest.java
new file mode 100644
index 0000000..f132089
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/AbstractSystemMultipartUploaderTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.tuple.Pair;
+
+import org.junit.Test;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public abstract class AbstractSystemMultipartUploaderTest {
+
+ abstract FileSystem getFS() throws IOException;
+
+ abstract Path getBaseTestPath();
+
+ @Test
+ public void testMultipartUpload() throws Exception {
+ FileSystem fs = getFS();
+ Path file = new Path(getBaseTestPath(), "some-file");
+ MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
+ UploadHandle uploadHandle = mpu.initialize(file);
+ List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ StringBuilder sb = new StringBuilder();
+ for (int i = 1; i <= 100; ++i) {
+ String contents = "ThisIsPart" + i + "\n";
+ sb.append(contents);
+ int len = contents.getBytes().length;
+ InputStream is = IOUtils.toInputStream(contents, "UTF-8");
+ PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle, len);
+ partHandles.add(Pair.of(i, partHandle));
+ }
+ PathHandle fd = mpu.complete(file, partHandles, uploadHandle);
+ byte[] fdData = IOUtils.toByteArray(fs.open(fd));
+ byte[] fileData = IOUtils.toByteArray(fs.open(file));
+ String readString = new String(fdData);
+ assertEquals(sb.toString(), readString);
+ assertArrayEquals(fdData, fileData);
+ }
+
+ @Test
+ public void testMultipartUploadReverseOrder() throws Exception {
+ FileSystem fs = getFS();
+ Path file = new Path(getBaseTestPath(), "some-file");
+ MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
+ UploadHandle uploadHandle = mpu.initialize(file);
+ List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ StringBuilder sb = new StringBuilder();
+ for (int i = 1; i <= 100; ++i) {
+ String contents = "ThisIsPart" + i + "\n";
+ sb.append(contents);
+ }
+ for (int i = 100; i > 0; --i) {
+ String contents = "ThisIsPart" + i + "\n";
+ int len = contents.getBytes().length;
+ InputStream is = IOUtils.toInputStream(contents, "UTF-8");
+ PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle, len);
+ partHandles.add(Pair.of(i, partHandle));
+ }
+ PathHandle fd = mpu.complete(file, partHandles, uploadHandle);
+ byte[] fdData = IOUtils.toByteArray(fs.open(fd));
+ byte[] fileData = IOUtils.toByteArray(fs.open(file));
+ String readString = new String(fdData);
+ assertEquals(sb.toString(), readString);
+ assertArrayEquals(fdData, fileData);
+ }
+
+ @Test
+ public void testMultipartUploadReverseOrderNoNContiguousPartNumbers()
+ throws Exception {
+ FileSystem fs = getFS();
+ Path file = new Path(getBaseTestPath(), "some-file");
+ MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
+ UploadHandle uploadHandle = mpu.initialize(file);
+ List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ StringBuilder sb = new StringBuilder();
+ for (int i = 2; i <= 200; i += 2) {
+ String contents = "ThisIsPart" + i + "\n";
+ sb.append(contents);
+ }
+ for (int i = 200; i > 0; i -= 2) {
+ String contents = "ThisIsPart" + i + "\n";
+ int len = contents.getBytes().length;
+ InputStream is = IOUtils.toInputStream(contents, "UTF-8");
+ PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle, len);
+ partHandles.add(Pair.of(i, partHandle));
+ }
+ PathHandle fd = mpu.complete(file, partHandles, uploadHandle);
+ byte[] fdData = IOUtils.toByteArray(fs.open(fd));
+ byte[] fileData = IOUtils.toByteArray(fs.open(file));
+ String readString = new String(fdData);
+ assertEquals(sb.toString(), readString);
+ assertArrayEquals(fdData, fileData);
+ }
+
+ @Test
+ public void testMultipartUploadAbort() throws Exception {
+ FileSystem fs = getFS();
+ Path file = new Path(getBaseTestPath(), "some-file");
+ MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
+ UploadHandle uploadHandle = mpu.initialize(file);
+ for (int i = 100; i >= 50; --i) {
+ String contents = "ThisIsPart" + i + "\n";
+ int len = contents.getBytes().length;
+ InputStream is = IOUtils.toInputStream(contents, "UTF-8");
+ PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle, len);
+ }
+ mpu.abort(file, uploadHandle);
+
+ String contents = "ThisIsPart49\n";
+ int len = contents.getBytes().length;
+ InputStream is = IOUtils.toInputStream(contents, "UTF-8");
+
+ try {
+ mpu.putPart(file, is, 49, uploadHandle, len);
+ fail("putPart should have thrown an exception");
+ } catch (IOException ok) {
+ // ignore
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemMultipartUploader.java
new file mode 100644
index 0000000..21d01b6
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemMultipartUploader.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.test.GenericTestUtils.getRandomizedTestDir;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Test the FileSystemMultipartUploader on local file system.
+ */
+public class TestLocalFileSystemMultipartUploader
+ extends AbstractSystemMultipartUploaderTest {
+
+ private static FileSystem fs;
+ private File tmp;
+
+ @BeforeClass
+ public static void init() throws IOException {
+ fs = LocalFileSystem.getLocal(new Configuration());
+ }
+
+ @Before
+ public void setup() throws IOException {
+ tmp = getRandomizedTestDir();
+ tmp.mkdirs();
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ tmp.delete();
+ }
+
+ @Override
+ public FileSystem getFS() {
+ return fs;
+ }
+
+ @Override
+ public Path getBaseTestPath() {
+ return new Path(tmp.getAbsolutePath());
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractPathHandleTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractPathHandleTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractPathHandleTest.java
index fbe28c3..36cfa6c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractPathHandleTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractPathHandleTest.java
@@ -123,6 +123,12 @@ public abstract class AbstractContractPathHandleTest
HandleOpt.Data data = HandleOpt.getOpt(HandleOpt.Data.class, opts)
.orElseThrow(IllegalArgumentException::new);
FileStatus stat = testFile(B1);
+ try {
+ // Temporary workaround while RawLocalFS supports only second precision
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
// modify the file by appending data
appendFile(getFileSystem(), stat.getPath(), B2);
byte[] b12 = Arrays.copyOf(B1, B1.length + B2.length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractPathHandle.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractPathHandle.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractPathHandle.java
new file mode 100644
index 0000000..3c088d2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractPathHandle.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.contract.rawlocal;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.contract.AbstractContractPathHandleTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.localfs.LocalFSContract;
+import org.apache.hadoop.fs.contract.rawlocal.RawlocalFSContract;
+
+public class TestRawlocalContractPathHandle
+ extends AbstractContractPathHandleTest {
+
+ public TestRawlocalContractPathHandle(String testname,
+ Options.HandleOpt[] opts, boolean serialized) {
+ super(testname, opts, serialized);
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new RawlocalFSContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-common-project/hadoop-common/src/test/resources/contract/rawlocal.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/contract/rawlocal.xml b/hadoop-common-project/hadoop-common/src/test/resources/contract/rawlocal.xml
index a0d1d21..8cbd4a0 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/contract/rawlocal.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/contract/rawlocal.xml
@@ -122,4 +122,9 @@
<value>true</value>
</property>
+ <property>
+ <name>fs.contract.supports-content-check</name>
+ <value>true</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSMultipartUploaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSMultipartUploaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSMultipartUploaderFactory.java
new file mode 100644
index 0000000..e9959c1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSMultipartUploaderFactory.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemMultipartUploader;
+import org.apache.hadoop.fs.MultipartUploader;
+import org.apache.hadoop.fs.MultipartUploaderFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+
+/**
+ * Support for HDFS multipart uploads, built on
+ * {@link FileSystem#concat(Path, Path[])}.
+ */
+public class DFSMultipartUploaderFactory extends MultipartUploaderFactory {
+ protected MultipartUploader createMultipartUploader(FileSystem fs,
+ Configuration conf) {
+ if (fs.getScheme().equals(HdfsConstants.HDFS_URI_SCHEME)) {
+ return new FileSystemMultipartUploader(fs);
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.fs.MultipartUploaderFactory
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.fs.MultipartUploaderFactory b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.fs.MultipartUploaderFactory
new file mode 100644
index 0000000..b153fd9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.fs.MultipartUploaderFactory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.hdfs.DFSMultipartUploaderFactory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSMultipartUploader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSMultipartUploader.java
new file mode 100644
index 0000000..96c5093
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSMultipartUploader.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+
+import java.io.IOException;
+
+public class TestHDFSMultipartUploader
+ extends AbstractSystemMultipartUploaderTest {
+
+ private static MiniDFSCluster cluster;
+ private Path tmp;
+
+ @Rule
+ public TestName name = new TestName();
+
+ @BeforeClass
+ public static void init() throws IOException {
+ HdfsConfiguration conf = new HdfsConfiguration();
+ cluster = new MiniDFSCluster.Builder(conf,
+ GenericTestUtils.getRandomizedTestDir())
+ .numDataNodes(1)
+ .build();
+ cluster.waitClusterUp();
+ }
+
+ @AfterClass
+ public static void cleanup() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ @Before
+ public void setup() throws IOException {
+ tmp = new Path(cluster.getFileSystem().getWorkingDirectory(),
+ name.getMethodName());
+ cluster.getFileSystem().mkdirs(tmp);
+ }
+
+ @Override
+ public FileSystem getFS() throws IOException {
+ return cluster.getFileSystem();
+ }
+
+ @Override
+ public Path getBaseTestPath() {
+ return tmp;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
new file mode 100644
index 0000000..34c88d4
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
+import com.amazonaws.services.s3.model.PartETag;
+import com.amazonaws.services.s3.model.UploadPartRequest;
+import com.amazonaws.services.s3.model.UploadPartResult;
+import com.google.common.base.Charsets;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BBPartHandle;
+import org.apache.hadoop.fs.BBUploadHandle;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.MultipartUploader;
+import org.apache.hadoop.fs.MultipartUploaderFactory;
+import org.apache.hadoop.fs.PartHandle;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathHandle;
+import org.apache.hadoop.fs.UploadHandle;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * MultipartUploader for S3AFileSystem. This uses the S3 multipart
+ * upload mechanism.
+ */
+public class S3AMultipartUploader extends MultipartUploader {
+
+ private final S3AFileSystem s3a;
+
+ public S3AMultipartUploader(FileSystem fs, Configuration conf) {
+ if (!(fs instanceof S3AFileSystem)) {
+ throw new IllegalArgumentException(
+ "S3A MultipartUploads must use S3AFileSystem");
+ }
+ s3a = (S3AFileSystem) fs;
+ }
+
+ @Override
+ public UploadHandle initialize(Path filePath) throws IOException {
+ String key = s3a.pathToKey(filePath);
+ InitiateMultipartUploadRequest request =
+ new InitiateMultipartUploadRequest(s3a.getBucket(), key);
+ LOG.debug("initialize request: {}", request);
+ InitiateMultipartUploadResult result = s3a.initiateMultipartUpload(request);
+ String uploadId = result.getUploadId();
+ return BBUploadHandle.from(ByteBuffer.wrap(
+ uploadId.getBytes(Charsets.UTF_8)));
+ }
+
+ @Override
+ public PartHandle putPart(Path filePath, InputStream inputStream,
+ int partNumber, UploadHandle uploadId, long lengthInBytes) {
+ String key = s3a.pathToKey(filePath);
+ UploadPartRequest request = new UploadPartRequest();
+ byte[] uploadIdBytes = uploadId.toByteArray();
+ request.setUploadId(new String(uploadIdBytes, 0, uploadIdBytes.length,
+ Charsets.UTF_8));
+ request.setInputStream(inputStream);
+ request.setPartSize(lengthInBytes);
+ request.setPartNumber(partNumber);
+ request.setBucketName(s3a.getBucket());
+ request.setKey(key);
+ LOG.debug("putPart request: {}", request);
+ UploadPartResult result = s3a.uploadPart(request);
+ String eTag = result.getETag();
+ return BBPartHandle.from(ByteBuffer.wrap(eTag.getBytes(Charsets.UTF_8)));
+ }
+
+ @Override
+ public PathHandle complete(Path filePath,
+ List<Pair<Integer, PartHandle>> handles, UploadHandle uploadId) {
+ String key = s3a.pathToKey(filePath);
+ CompleteMultipartUploadRequest request =
+ new CompleteMultipartUploadRequest();
+ request.setBucketName(s3a.getBucket());
+ request.setKey(key);
+ byte[] uploadIdBytes = uploadId.toByteArray();
+ request.setUploadId(new String(uploadIdBytes, 0, uploadIdBytes.length,
+ Charsets.UTF_8));
+ List<PartETag> eTags = handles
+ .stream()
+ .map(handle -> {
+ byte[] partEtagBytes = handle.getRight().toByteArray();
+ return new PartETag(handle.getLeft(),
+ new String(partEtagBytes, 0, partEtagBytes.length,
+ Charsets.UTF_8));
+ })
+ .collect(Collectors.toList());
+ request.setPartETags(eTags);
+ LOG.debug("Complete request: {}", request);
+ CompleteMultipartUploadResult completeMultipartUploadResult =
+ s3a.getAmazonS3Client().completeMultipartUpload(request);
+
+ byte[] eTag = DFSUtilClient.string2Bytes(
+ completeMultipartUploadResult.getETag());
+ return (PathHandle) () -> ByteBuffer.wrap(eTag);
+ }
+
+ @Override
+ public void abort(Path filePath, UploadHandle uploadId) {
+ String key = s3a.pathToKey(filePath);
+ byte[] uploadIdBytes = uploadId.toByteArray();
+ String uploadIdString = new String(uploadIdBytes, 0, uploadIdBytes.length,
+ Charsets.UTF_8);
+ AbortMultipartUploadRequest request = new AbortMultipartUploadRequest(s3a
+ .getBucket(), key, uploadIdString);
+ LOG.debug("Abort request: {}", request);
+ s3a.getAmazonS3Client().abortMultipartUpload(request);
+ }
+
+ /**
+ * Factory for creating MultipartUploader objects for s3a:// FileSystems.
+ */
+ public static class Factory extends MultipartUploaderFactory {
+ @Override
+ protected MultipartUploader createMultipartUploader(FileSystem fs,
+ Configuration conf) {
+ if (fs.getScheme().equals("s3a")) {
+ return new S3AMultipartUploader(fs, conf);
+ }
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-tools/hadoop-aws/src/main/resources/META-INF/org.apache.hadoop.fs.MultipartUploaderFactory
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/org.apache.hadoop.fs.MultipartUploaderFactory b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/org.apache.hadoop.fs.MultipartUploaderFactory
new file mode 100644
index 0000000..2e4bc24
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/org.apache.hadoop.fs.MultipartUploaderFactory
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.hadoop.fs.s3a.S3AMultipartUploader$Factory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980031bb/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.MultipartUploader
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.MultipartUploader b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.MultipartUploader
new file mode 100644
index 0000000..d16846b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.MultipartUploader
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.fs.s3a.S3AMultipartUploader
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HDFS-13686. Add overall metrics
for FSNamesystemLock. Contributed by Lukas Majercak.
Posted by xk...@apache.org.
HDFS-13686. Add overall metrics for FSNamesystemLock. Contributed by Lukas Majercak.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d31a3ce7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d31a3ce7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d31a3ce7
Branch: refs/heads/HDFS-12943
Commit: d31a3ce767d3bb68bdbb4f36d45600eab9f4f8b7
Parents: 1da3b55
Author: Inigo Goiri <in...@apache.org>
Authored: Fri Jun 15 15:59:19 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Fri Jun 15 15:59:19 2018 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/FSNamesystemLock.java | 17 ++++++++++++-----
.../hdfs/server/namenode/TestFSNamesystemLock.java | 10 ++++++++--
2 files changed, 20 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d31a3ce7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
index 900f8a2..f8e69e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
@@ -107,6 +107,8 @@ class FSNamesystemLock {
private static final String WRITE_LOCK_METRIC_PREFIX = "FSNWriteLock";
private static final String LOCK_METRIC_SUFFIX = "Nanos";
+ private static final String OVERALL_METRIC_NAME = "Overall";
+
FSNamesystemLock(Configuration conf,
MutableRatesWithAggregation detailedHoldTimeMetrics) {
this(conf, detailedHoldTimeMetrics, new Timer());
@@ -320,12 +322,17 @@ class FSNamesystemLock {
*/
private void addMetric(String operationName, long value, boolean isWrite) {
if (metricsEnabled) {
- String metricName =
- (isWrite ? WRITE_LOCK_METRIC_PREFIX : READ_LOCK_METRIC_PREFIX) +
- org.apache.commons.lang.StringUtils.capitalize(operationName) +
- LOCK_METRIC_SUFFIX;
- detailedHoldTimeMetrics.add(metricName, value);
+ String opMetric = getMetricName(operationName, isWrite);
+ detailedHoldTimeMetrics.add(opMetric, value);
+
+ String overallMetric = getMetricName(OVERALL_METRIC_NAME, isWrite);
+ detailedHoldTimeMetrics.add(overallMetric, value);
}
}
+ private static String getMetricName(String operationName, boolean isWrite) {
+ return (isWrite ? WRITE_LOCK_METRIC_PREFIX : READ_LOCK_METRIC_PREFIX) +
+ org.apache.commons.lang.StringUtils.capitalize(operationName) +
+ LOCK_METRIC_SUFFIX;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d31a3ce7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
index 2daf5c2..49506fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
@@ -333,7 +333,7 @@ public class TestFSNamesystemLock {
FSNamesystemLock fsLock = new FSNamesystemLock(conf, rates, timer);
fsLock.readLock();
- timer.advanceNanos(1200000);
+ timer.advanceNanos(1300000);
fsLock.readUnlock("foo");
fsLock.readLock();
timer.advanceNanos(2400000);
@@ -353,12 +353,18 @@ public class TestFSNamesystemLock {
MetricsRecordBuilder rb = MetricsAsserts.mockMetricsRecordBuilder();
rates.snapshot(rb, true);
- assertGauge("FSNReadLockFooNanosAvgTime", 1800000.0, rb);
+ assertGauge("FSNReadLockFooNanosAvgTime", 1850000.0, rb);
assertCounter("FSNReadLockFooNanosNumOps", 2L, rb);
assertGauge("FSNReadLockBarNanosAvgTime", 2000000.0, rb);
assertCounter("FSNReadLockBarNanosNumOps", 1L, rb);
assertGauge("FSNWriteLockBazNanosAvgTime", 1000000.0, rb);
assertCounter("FSNWriteLockBazNanosNumOps", 1L, rb);
+
+ // Overall
+ assertGauge("FSNReadLockOverallNanosAvgTime", 1900000.0, rb);
+ assertCounter("FSNReadLockOverallNanosNumOps", 3L, rb);
+ assertGauge("FSNWriteLockOverallNanosAvgTime", 1000000.0, rb);
+ assertCounter("FSNWriteLockOverallNanosNumOps", 1L, rb);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: YARN-8438.
TestContainer.testKillOnNew flaky on trunk. Contributed by Szilard Nemeth.
Posted by xk...@apache.org.
YARN-8438. TestContainer.testKillOnNew flaky on trunk. Contributed by Szilard Nemeth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35ec9401
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35ec9401
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35ec9401
Branch: refs/heads/HDFS-12943
Commit: 35ec9401e829bfa10994790659a26b0babacae35
Parents: 7a3c6e9
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Mon Jun 25 09:23:11 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Mon Jun 25 15:47:54 2018 -0700
----------------------------------------------------------------------
.../containermanager/container/TestContainer.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35ec9401/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index 1a263ee..edf26d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -116,7 +116,7 @@ public class TestContainer {
final NodeManagerMetrics metrics = NodeManagerMetrics.create();
final Configuration conf = new YarnConfiguration();
final String FAKE_LOCALIZATION_ERROR = "Fake localization error";
-
+
/**
* Verify correct container request events sent to localizer.
*/
@@ -591,9 +591,8 @@ public class TestContainer {
Assert.assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER,
containerMetrics.exitCode.value());
Assert.assertTrue(containerMetrics.startTime.value() > 0);
- Assert.assertTrue(
- containerMetrics.finishTime.value() > containerMetrics.startTime
- .value());
+ Assert.assertTrue(containerMetrics.finishTime.value() >=
+ containerMetrics.startTime.value());
Assert.assertEquals(ContainerEventType.KILL_CONTAINER,
wc.initStateToEvent.get(ContainerState.NEW));
Assert.assertEquals(ContainerState.DONE,
@@ -1612,4 +1611,5 @@ public class TestContainer {
return ((ContainerImpl)c).getRetryPolicy();
}
}
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: YARN-8457. Compilation is broken
with -Pyarn-ui.
Posted by xk...@apache.org.
YARN-8457. Compilation is broken with -Pyarn-ui.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ffe68a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ffe68a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ffe68a6
Branch: refs/heads/HDFS-12943
Commit: 4ffe68a6f70ce01a5654da8991b4cdb35ae0bf1f
Parents: abc3e4b
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Mon Jun 25 10:38:03 2018 -0700
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Mon Jun 25 10:38:03 2018 -0700
----------------------------------------------------------------------
.../hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ffe68a6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
index 959e169..daf4462 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
@@ -1,4 +1,5 @@
{
"directory": "bower_components",
- "analytics": false
+ "analytics": false,
+ "registry": "https://registry.bower.io"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDFS-13174. hdfs mover -p /path
times out after 20 min. Contributed by Istvan Fajth.
Posted by xk...@apache.org.
HDFS-13174. hdfs mover -p /path times out after 20 min. Contributed by Istvan Fajth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c966a383
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c966a383
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c966a383
Branch: refs/heads/HDFS-12943
Commit: c966a3837af1c1a1c4a441f491b0d76d5c9e5d78
Parents: eebeb60
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Fri Jun 15 13:35:50 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Fri Jun 15 13:36:46 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 6 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 30 +++++---
.../src/main/resources/hdfs-default.xml | 10 +++
.../hdfs/server/balancer/TestBalancer.java | 79 ++++++++++++++++++++
.../hadoop/hdfs/server/mover/TestMover.java | 46 ++++++++++++
6 files changed, 163 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c966a383/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index bc8e81f..dde7eb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -581,7 +581,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_BALANCER_BLOCK_MOVE_TIMEOUT = "dfs.balancer.block-move.timeout";
public static final int DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT = 0;
public static final String DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY = "dfs.balancer.max-no-move-interval";
- public static final int DFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT = 60*1000; // One minute
+ public static final int DFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT = 60*1000; // One minute
+ public static final String DFS_BALANCER_MAX_ITERATION_TIME_KEY = "dfs.balancer.max-iteration-time";
+ public static final long DFS_BALANCER_MAX_ITERATION_TIME_DEFAULT = 20 * 60 * 1000L; // 20 mins
public static final String DFS_MOVER_MOVEDWINWIDTH_KEY = "dfs.mover.movedWinWidth";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c966a383/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 13d5846..426c7ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -289,13 +289,17 @@ public class Balancer {
final int maxNoMoveInterval = conf.getInt(
DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY,
DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT);
+ final long maxIterationTime = conf.getLong(
+ DFSConfigKeys.DFS_BALANCER_MAX_ITERATION_TIME_KEY,
+ DFSConfigKeys.DFS_BALANCER_MAX_ITERATION_TIME_DEFAULT);
this.nnc = theblockpool;
this.dispatcher =
new Dispatcher(theblockpool, p.getIncludedNodes(),
p.getExcludedNodes(), movedWinWidth, moverThreads,
dispatcherThreads, maxConcurrentMovesPerNode, getBlocksSize,
- getBlocksMinBlockSize, blockMoveTimeout, maxNoMoveInterval, conf);
+ getBlocksMinBlockSize, blockMoveTimeout, maxNoMoveInterval,
+ maxIterationTime, conf);
this.threshold = p.getThreshold();
this.policy = p.getBalancingPolicy();
this.sourceNodes = p.getSourceNodes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c966a383/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index 349ced1..060c013 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@ -138,6 +138,8 @@ public class Dispatcher {
private final boolean connectToDnViaHostname;
private BlockPlacementPolicies placementPolicies;
+ private long maxIterationTime;
+
static class Allocator {
private final int max;
private int count = 0;
@@ -346,13 +348,19 @@ public class Dispatcher {
/** Dispatch the move to the proxy source & wait for the response. */
private void dispatch() {
- LOG.info("Start moving " + this);
- assert !(reportedBlock instanceof DBlockStriped);
-
Socket sock = new Socket();
DataOutputStream out = null;
DataInputStream in = null;
try {
+ if (source.isIterationOver()){
+ LOG.info("Cancel moving " + this +
+ " as iteration is already cancelled due to" +
+ " dfs.balancer.max-iteration-time is passed.");
+ throw new IOException("Block move cancelled.");
+ }
+ LOG.info("Start moving " + this);
+ assert !(reportedBlock instanceof DBlockStriped);
+
sock.connect(
NetUtils.createSocketAddr(target.getDatanodeInfo().
getXferAddr(Dispatcher.this.connectToDnViaHostname)),
@@ -760,7 +768,10 @@ public class Dispatcher {
* Check if the iteration is over
*/
public boolean isIterationOver() {
- return (Time.monotonicNow()-startTime > MAX_ITERATION_TIME);
+ if (maxIterationTime < 0){
+ return false;
+ }
+ return (Time.monotonicNow()-startTime > maxIterationTime);
}
/** Add a task */
@@ -908,8 +919,6 @@ public class Dispatcher {
return blocksToReceive > 0;
}
- private static final long MAX_ITERATION_TIME = 20 * 60 * 1000L; // 20 mins
-
/**
* This method iteratively does the following: it first selects a block to
* move, then sends a request to the proxy source to start the block move
@@ -990,7 +999,7 @@ public class Dispatcher {
}
if (isIterationOver()) {
- LOG.info("The maximum iteration time (" + MAX_ITERATION_TIME/1000
+ LOG.info("The maximum iteration time (" + maxIterationTime/1000
+ " seconds) has been reached. Stopping " + this);
}
}
@@ -1013,14 +1022,14 @@ public class Dispatcher {
int maxNoMoveInterval, Configuration conf) {
this(nnc, includedNodes, excludedNodes, movedWinWidth,
moverThreads, dispatcherThreads, maxConcurrentMovesPerNode,
- 0L, 0L, 0, maxNoMoveInterval, conf);
+ 0L, 0L, 0, maxNoMoveInterval, -1, conf);
}
Dispatcher(NameNodeConnector nnc, Set<String> includedNodes,
Set<String> excludedNodes, long movedWinWidth, int moverThreads,
int dispatcherThreads, int maxConcurrentMovesPerNode,
- long getBlocksSize, long getBlocksMinBlockSize,
- int blockMoveTimeout, int maxNoMoveInterval, Configuration conf) {
+ long getBlocksSize, long getBlocksMinBlockSize, int blockMoveTimeout,
+ int maxNoMoveInterval, long maxIterationTime, Configuration conf) {
this.nnc = nnc;
this.excludedNodes = excludedNodes;
this.includedNodes = includedNodes;
@@ -1047,6 +1056,7 @@ public class Dispatcher {
HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME,
HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
placementPolicies = new BlockPlacementPolicies(conf, null, cluster, null);
+ this.maxIterationTime = maxIterationTime;
}
public DistributedFileSystem getDistributedFileSystem() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c966a383/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index b55421c..146ae6c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3541,6 +3541,16 @@
</property>
<property>
+ <name>dfs.balancer.max-iteration-time</name>
+ <value>1200000</value>
+ <description>
+ Maximum amount of time while an iteration can be run by the Balancer. After
+ this time the Balancer will stop the iteration, and reevaluate the work
+ needs to be done to Balance the cluster. The default value is 20 minutes.
+ </description>
+</property>
+
+<property>
<name>dfs.block.invalidate.limit</name>
<value>1000</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c966a383/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 9579b82..fa026f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -1580,6 +1580,85 @@ public class TestBalancer {
CAPACITY, RACK2, new PortNumberBasedNodes(3, 0, 1), true, true);
}
+
+ @Test(timeout = 100000)
+ public void testMaxIterationTime() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ int blockSize = 10*1024*1024; // 10MB block size
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, blockSize);
+ // limit the worker thread count of Balancer to have only 1 queue per DN
+ conf.setInt(DFSConfigKeys.DFS_BALANCER_MOVERTHREADS_KEY, 1);
+ // limit the bandwitdh to 1 packet per sec to emulate slow block moves
+ conf.setLong(DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY,
+ 64 * 1024);
+ // set client socket timeout to have an IN_PROGRESS notification back from
+ // the DataNode about the copy in every second.
+ conf.setLong(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 2000L);
+ // set max iteration time to 2 seconds to timeout before moving any block
+ conf.setLong(DFSConfigKeys.DFS_BALANCER_MAX_ITERATION_TIME_KEY, 2000L);
+ // setup the cluster
+ final long capacity = 10L * blockSize;
+ final long[] dnCapacities = new long[] {capacity, capacity};
+ final short rep = 1;
+ final long seed = 0xFAFAFA;
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(0)
+ .build();
+ try {
+ cluster.getConfiguration(0).setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+ cluster.startDataNodes(conf, 1, true, null, null, dnCapacities);
+ cluster.waitClusterUp();
+ cluster.waitActive();
+ final Path path = new Path("/testMaxIterationTime.dat");
+ DistributedFileSystem fs = cluster.getFileSystem();
+ // fill the DN to 40%
+ DFSTestUtil.createFile(fs, path, 4L * blockSize, rep, seed);
+ // start a new DN
+ cluster.startDataNodes(conf, 1, true, null, null, dnCapacities);
+ cluster.triggerHeartbeats();
+ // setup Balancer and run one iteration
+ List<NameNodeConnector> connectors = Collections.emptyList();
+ try {
+ BalancerParameters bParams = BalancerParameters.DEFAULT;
+ connectors = NameNodeConnector.newNameNodeConnectors(
+ DFSUtil.getInternalNsRpcUris(conf), Balancer.class.getSimpleName(),
+ Balancer.BALANCER_ID_PATH, conf, bParams.getMaxIdleIteration());
+ for (NameNodeConnector nnc : connectors) {
+ LOG.info("NNC to work on: " + nnc);
+ Balancer b = new Balancer(nnc, bParams, conf);
+ long startTime = Time.monotonicNow();
+ Result r = b.runOneIteration();
+ long runtime = Time.monotonicNow() - startTime;
+ assertEquals("We expect ExitStatus.IN_PROGRESS to be reported.",
+ ExitStatus.IN_PROGRESS, r.exitStatus);
+ // accept runtime if it is under 3.5 seconds, as we need to wait for
+ // IN_PROGRESS report from DN, and some spare to be able to finish.
+ // NOTE: This can be a source of flaky tests, if the box is busy,
+ // assertion here is based on the following: Balancer is already set
+ // up, iteration gets the blocks from the NN, and makes the decision
+ // to move 2 blocks. After that the PendingMoves are scheduled, and
+ // DataNode heartbeats in for the Balancer every second, iteration is
+ // two seconds long. This means that it will fail if the setup and the
+ // heartbeat from the DataNode takes more than 500ms, as the iteration
+ // should end at the 3rd second from start. As the number of
+ // operations seems to be pretty low, and all comm happens locally, I
+ // think the possibility of a failure due to node busyness is low.
+ assertTrue("Unexpected iteration runtime: " + runtime + "ms > 3.5s",
+ runtime < 3500);
+ }
+ } finally {
+ for (NameNodeConnector nnc : connectors) {
+ IOUtils.cleanupWithLogger(null, nnc);
+ }
+ }
+ } finally {
+ cluster.shutdown(true, true);
+ }
+ }
+
/*
* Test Balancer with Ram_Disk configured
* One DN has two files on RAM_DISK, other DN has no files on RAM_DISK.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c966a383/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index 36e7bb9..62c91bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -685,6 +685,52 @@ public class TestMover {
}
}
+ @Test(timeout=100000)
+ public void testBalancerMaxIterationTimeNotAffectMover() throws Exception {
+ long blockSize = 10*1024*1024;
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ conf.setInt(DFSConfigKeys.DFS_MOVER_MOVERTHREADS_KEY, 1);
+ conf.setInt(
+ DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY, 1);
+ // set a fairly large block size to run into the limitation
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ conf.setLong(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, blockSize);
+ // set a somewhat grater than zero max iteration time to have the move time
+ // to surely exceed it
+ conf.setLong(DFSConfigKeys.DFS_BALANCER_MAX_ITERATION_TIME_KEY, 200L);
+ conf.setInt(DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY, 1);
+ // set client socket timeout to have an IN_PROGRESS notification back from
+ // the DataNode about the copy in every second.
+ conf.setLong(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 1000L);
+
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(2)
+ .storageTypes(
+ new StorageType[][] {{StorageType.DISK, StorageType.DISK},
+ {StorageType.ARCHIVE, StorageType.ARCHIVE}})
+ .build();
+ try {
+ cluster.waitActive();
+ final DistributedFileSystem fs = cluster.getFileSystem();
+ final String file = "/testMaxIterationTime.dat";
+ final Path path = new Path(file);
+ short rep_factor = 1;
+ int seed = 0xFAFAFA;
+ // write to DISK
+ DFSTestUtil.createFile(fs, path, 4L * blockSize, rep_factor, seed);
+
+ // move to ARCHIVE
+ fs.setStoragePolicy(new Path(file), "COLD");
+ int rc = ToolRunner.run(conf, new Mover.Cli(),
+ new String[] {"-p", file});
+ Assert.assertEquals("Retcode expected to be ExitStatus.SUCCESS (0).",
+ ExitStatus.SUCCESS.getExitCode(), rc);
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
private final ErasureCodingPolicy ecPolicy =
StripedFileTestUtil.getDefaultECPolicy();
private final int dataBlocks = ecPolicy.getNumDataUnits();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HDDS-172. The numbers of operation
should be integer in KSM UI. Contributed by Takanobu Asanuma.
Posted by xk...@apache.org.
HDDS-172. The numbers of operation should be integer in KSM UI.
Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/308a1591
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/308a1591
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/308a1591
Branch: refs/heads/HDFS-12943
Commit: 308a1591f9f41597f4e7cc17bca06c66d6efc0a2
Parents: c966a38
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Jun 15 10:23:58 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Fri Jun 15 14:10:17 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/ozone-manager/src/main/webapps/ksm/ksm.js | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308a1591/hadoop-ozone/ozone-manager/src/main/webapps/ksm/ksm.js
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ksm/ksm.js b/hadoop-ozone/ozone-manager/src/main/webapps/ksm/ksm.js
index 7fb52b1..ab6f73b 100644
--- a/hadoop-ozone/ozone-manager/src/main/webapps/ksm/ksm.js
+++ b/hadoop-ozone/ozone-manager/src/main/webapps/ksm/ksm.js
@@ -48,7 +48,9 @@
labelType: 'value',
duration: 500,
labelThreshold: 0.01,
- labelSunbeamLayout: true,
+ valueFormat: function(d) {
+ return d3.format('d')(d);
+ },
legend: {
margin: {
top: 5,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HDDS-177. Create a releasable
ozonefs artifact Contributed by Marton, Elek.
Posted by xk...@apache.org.
HDDS-177. Create a releasable ozonefs artifact
Contributed by Marton, Elek.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e16e5b30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e16e5b30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e16e5b30
Branch: refs/heads/HDFS-12943
Commit: e16e5b307d6c4404db0698b9d128e5bf4aa16a8a
Parents: ca14fec
Author: Anu Engineer <ae...@apache.org>
Authored: Sun Jun 24 01:05:04 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Sun Jun 24 01:05:04 2018 -0700
----------------------------------------------------------------------
dev-support/bin/ozone-dist-layout-stitching | 4 +-
hadoop-dist/pom.xml | 5 +
.../test/acceptance/ozonefs/docker-compose.yaml | 71 ++
.../src/test/acceptance/ozonefs/docker-config | 39 ++
.../src/test/acceptance/ozonefs/ozonefs.robot | 39 ++
hadoop-ozone/ozonefs/pom.xml | 211 ++++++
.../org/apache/hadoop/fs/ozone/Constants.java | 42 ++
.../java/org/apache/hadoop/fs/ozone/OzFs.java | 44 ++
.../hadoop/fs/ozone/OzoneFSInputStream.java | 79 +++
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 59 ++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 +++++++++++++++++++
.../apache/hadoop/fs/ozone/package-info.java | 30 +
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 +++++
.../fs/ozone/TestOzoneFileInterfaces.java | 231 +++++++
.../contract/ITestOzoneContractCreate.java | 48 ++
.../contract/ITestOzoneContractDelete.java | 48 ++
.../contract/ITestOzoneContractDistCp.java | 50 ++
.../ITestOzoneContractGetFileStatus.java | 61 ++
.../ozone/contract/ITestOzoneContractMkdir.java | 48 ++
.../ozone/contract/ITestOzoneContractOpen.java | 47 ++
.../contract/ITestOzoneContractRename.java | 49 ++
.../contract/ITestOzoneContractRootDir.java | 51 ++
.../ozone/contract/ITestOzoneContractSeek.java | 47 ++
.../hadoop/fs/ozone/contract/OzoneContract.java | 123 ++++
.../src/test/resources/contract/ozone.xml | 113 +++
.../ozonefs/src/test/resources/log4j.properties | 23 +
hadoop-ozone/pom.xml | 1 +
hadoop-project/pom.xml | 6 +-
hadoop-tools/hadoop-ozone/pom.xml | 174 -----
.../org/apache/hadoop/fs/ozone/Constants.java | 42 --
.../java/org/apache/hadoop/fs/ozone/OzFs.java | 44 --
.../hadoop/fs/ozone/OzoneFSInputStream.java | 79 ---
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 59 --
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 -------------------
.../apache/hadoop/fs/ozone/package-info.java | 30 -
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 -----
.../fs/ozone/TestOzoneFileInterfaces.java | 231 -------
.../contract/ITestOzoneContractCreate.java | 48 --
.../contract/ITestOzoneContractDelete.java | 48 --
.../contract/ITestOzoneContractDistCp.java | 50 --
.../ITestOzoneContractGetFileStatus.java | 61 --
.../ozone/contract/ITestOzoneContractMkdir.java | 48 --
.../ozone/contract/ITestOzoneContractOpen.java | 47 --
.../contract/ITestOzoneContractRename.java | 49 --
.../contract/ITestOzoneContractRootDir.java | 51 --
.../ozone/contract/ITestOzoneContractSeek.java | 47 --
.../hadoop/fs/ozone/contract/OzoneContract.java | 123 ----
.../src/test/resources/contract/ozone.xml | 113 ---
.../src/test/resources/log4j.properties | 23 -
hadoop-tools/hadoop-tools-dist/pom.xml | 15 -
hadoop-tools/pom.xml | 11 -
51 files changed, 2413 insertions(+), 2241 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/dev-support/bin/ozone-dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching
index ad8abe2..be330d5 100755
--- a/dev-support/bin/ozone-dist-layout-stitching
+++ b/dev-support/bin/ozone-dist-layout-stitching
@@ -145,6 +145,8 @@ run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-$
run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}" .
run copy "${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" .
run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" .
+mkdir -p "./share/hadoop/ozonefs"
+cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem.jar"
# Optional documentation, could be missing
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ksm/
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/
@@ -153,5 +155,5 @@ cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdd
mkdir -p ./share/hadoop/mapreduce
mkdir -p ./share/hadoop/yarn
echo
-echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone-${HDDS_VERSION}"
+echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone"
echo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index dfbf818..5de6759 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -265,6 +265,11 @@
<artifactId>hadoop-ozone-docs</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
new file mode 100644
index 0000000..3323557
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ namenode:
+ image: apache/hadoop-runner
+ hostname: namenode
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9870
+ environment:
+ ENSURE_NAMENODE_DIR: /data/namenode
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/hdfs","namenode"]
+ datanode:
+ image: apache/hadoop-runner
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ ksm:
+ image: apache/hadoop-runner
+ hostname: ksm
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9874
+ environment:
+ ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","ksm"]
+ scm:
+ image: apache/hadoop-runner
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
+ hadooplast:
+ image: flokkr/hadoop:3.1.0
+ volumes:
+ - ${OZONEDIR}:/opt/ozone
+ env_file:
+ - ./docker-config
+ environment:
+ HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/hadoop-ozone-filesystem.jar
+ command: ["watch","-n","100000","ls"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
new file mode 100644
index 0000000..dec863e
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
+CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
+OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.category.org.apache.hadoop.util.NativeCodeLoader=ERROR
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot
new file mode 100644
index 0000000..9e8a5d2
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Ozonefs test
+Library OperatingSystem
+Suite Setup Startup Ozone cluster with size 5
+Suite Teardown Teardown Ozone cluster
+Resource ../commonlib.robot
+
+*** Variables ***
+${COMPOSEFILE} ${CURDIR}/docker-compose.yaml
+${PROJECTDIR} ${CURDIR}/../../../../../..
+
+
+*** Test Cases ***
+Create volume and bucket
+ Execute on datanode ozone oz -createVolume http://ksm/fstest -user bilbo -quota 100TB -root
+ Execute on datanode ozone oz -createBucket http://ksm/fstest/bucket1
+
+Check volume from ozonefs
+ ${result} = Execute on hadooplast hdfs dfs -ls o3://bucket1.fstest/
+
+Create directory from ozonefs
+ Execute on hadooplast hdfs dfs -mkdir -p o3://bucket1.fstest/testdir/deep
+ ${result} = Execute on ksm ozone oz -listKey o3://ksm/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} testdir/deep
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml
new file mode 100644
index 0000000..c3de4d1
--- /dev/null
+++ b/hadoop-ozone/ozonefs/pom.xml
@@ -0,0 +1,211 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-ozone-filesystem</artifactId>
+ <name>Apache Hadoop Ozone FileSystem</name>
+ <packaging>jar</packaging>
+ <version>0.2.1-SNAPSHOT</version>
+ <properties>
+ <file.encoding>UTF-8</file.encoding>
+ <downloadSources>true</downloadSources>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.1.1</version>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>com.google.guava:guava:jar</include>
+ <include>org.slf4j:slf4j-api:jar</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>com.nimbusds:nimbus-jose-jwt:jar</include>
+ <include>com.github.stephenc.jcip:jcip-annotations</include>
+ <include>com.google.code.findbugs:jsr305:jar</include>
+ <include>org.apache.hadoop:hadoop-ozone-client</include>
+ <include>org.apache.hadoop:hadoop-hdds-client</include>
+ <include>org.apache.hadoop:hadoop-hdds-common</include>
+ <include>org.fusesource.leveldbjni:leveldbjni-all</include>
+ <include>org.apache.ratis:ratis-server</include>
+ <include>org.apache.ratis:ratis-proto-shaded:jar</include>
+ <include>com.google.auto.value:auto-value-annotations</include>
+ <include>com.squareup:javapoet:jar</include>
+ <include>org.jctools:jctools-core</include>
+ <include>org.apache.ratis:ratis-common</include>
+ <include>org.apache.ratis:ratis-client</include>
+ <include>org.apache.ratis:ratis-netty</include>
+ <include>org.apache.ratis:ratis-grpc</include>
+ <include>org.rocksdb:rocksdbjni</include>
+ <include>org.apache.hadoop:hadoop-ozone-common</include>
+ </includes>
+ </artifactSet>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>deplist</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>list</goal>
+ </goals>
+ <configuration>
+ <!-- build a shellprofile -->
+ <outputFile>
+ ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt
+ </outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-server-scm</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-server-framework</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-ozone-manager</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-container-service</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-objectstore-service</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-integration-test</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/Constants.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
new file mode 100644
index 0000000..832a0cb
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+/**
+ * Constants for Ozone FileSystem implementation.
+ */
+public final class Constants {
+
+ public static final String OZONE_DEFAULT_USER = "hdfs";
+
+ public static final String OZONE_USER_DIR = "/user";
+
+ /** Local buffer directory. */
+ public static final String BUFFER_DIR_KEY = "fs.ozone.buffer.dir";
+
+ /** Temporary directory. */
+ public static final String BUFFER_TMP_KEY = "hadoop.tmp.dir";
+
+ /** Page size for Ozone listing operation. */
+ public static final int LISTING_PAGE_SIZE = 1024;
+
+ private Constants() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
new file mode 100644
index 0000000..4163c13
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * ozone implementation of AbstractFileSystem.
+ * This impl delegates to the OzoneFileSystem
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class OzFs extends DelegateToFileSystem {
+
+ public OzFs(URI theUri, Configuration conf)
+ throws IOException, URISyntaxException {
+ super(theUri, new OzoneFileSystem(), conf,
+ OzoneConsts.OZONE_URI_SCHEME, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
new file mode 100644
index 0000000..4c5c0c8
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * The input stream for Ozone file system.
+ *
+ * TODO: Make inputStream generic for both rest and rpc clients
+ * This class is not thread safe.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class OzoneFSInputStream extends FSInputStream {
+
+ private final ChunkGroupInputStream inputStream;
+
+ public OzoneFSInputStream(InputStream inputStream) {
+ this.inputStream = (ChunkGroupInputStream)inputStream;
+ }
+
+ @Override
+ public int read() throws IOException {
+ return inputStream.read();
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ return inputStream.read(b, off, len);
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ inputStream.close();
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ inputStream.seek(pos);
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return inputStream.getPos();
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ return false;
+ }
+
+ @Override
+ public int available() throws IOException {
+ return inputStream.available();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
new file mode 100644
index 0000000..faa3628
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
+
+
+/**
+ * The output stream for Ozone file system.
+ *
+ * TODO: Make outputStream generic for both rest and rpc clients
+ * This class is not thread safe.
+ */
+public class OzoneFSOutputStream extends OutputStream {
+
+ private final ChunkGroupOutputStream outputStream;
+
+ public OzoneFSOutputStream(OutputStream outputStream) {
+ this.outputStream = (ChunkGroupOutputStream)outputStream;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ outputStream.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ outputStream.write(b, off, len);
+ }
+
+ @Override
+ public synchronized void flush() throws IOException {
+ outputStream.flush();
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ outputStream.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
new file mode 100644
index 0000000..6906a9d
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -0,0 +1,689 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Iterator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+
+/**
+ * The Ozone Filesystem implementation.
+ *
+ * This subclass is marked as private as code should not be creating it
+ * directly; use {@link FileSystem#get(Configuration)} and variants to create
+ * one. If cast to {@link OzoneFileSystem}, extra methods and features may be
+ * accessed. Consider those private and unstable.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class OzoneFileSystem extends FileSystem {
+ static final Logger LOG = LoggerFactory.getLogger(OzoneFileSystem.class);
+
+ /** The Ozone client for connecting to Ozone server. */
+ private OzoneClient ozoneClient;
+ private ObjectStore objectStore;
+ private OzoneVolume volume;
+ private OzoneBucket bucket;
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+
+ private static final Pattern URL_SCHEMA_PATTERN =
+ Pattern.compile("(.+)\\.([^\\.]+)");
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ assert getScheme().equals(name.getScheme());
+
+ String authority = name.getAuthority();
+
+ Matcher matcher = URL_SCHEMA_PATTERN.matcher(authority);
+
+ if (!matcher.matches()) {
+ throw new IllegalArgumentException("Ozone file system url should be "
+ + "in the form o3://bucket.volume");
+ }
+ String bucketStr = matcher.group(1);
+ String volumeStr = matcher.group(2);
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_URI_SCHEME)
+ .setHost(authority).build();
+ LOG.trace("Ozone URI for ozfs initialization is " + uri);
+ this.ozoneClient = OzoneClientFactory.getRpcClient(conf);
+ objectStore = ozoneClient.getObjectStore();
+ this.volume = objectStore.getVolume(volumeStr);
+ this.bucket = volume.getBucket(bucketStr);
+ this.replicationType = ReplicationType.valueOf(
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT));
+ this.replicationFactor = ReplicationFactor.valueOf(
+ conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT));
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ ozoneClient.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path f, int bufferSize) throws IOException {
+ LOG.trace("open() path:{}", f);
+ final FileStatus fileStatus = getFileStatus(f);
+ final String key = pathToKey(f);
+ if (fileStatus.isDirectory()) {
+ throw new FileNotFoundException("Can't open directory " + f + " to read");
+ }
+
+ return new FSDataInputStream(
+ new OzoneFSInputStream(bucket.readKey(key).getInputStream()));
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ final String key = pathToKey(f);
+ final FileStatus status;
+ try {
+ status = getFileStatus(f);
+ if (status.isDirectory()) {
+ throw new FileAlreadyExistsException(f + " is a directory");
+ } else {
+ if (!overwrite) {
+ // path references a file and overwrite is disabled
+ throw new FileAlreadyExistsException(f + " already exists");
+ }
+ LOG.trace("Overwriting file {}", f);
+ deleteObject(key);
+ }
+ } catch (FileNotFoundException ignored) {
+ // check if the parent directory needs to be created
+ Path parent = f.getParent();
+ try {
+ // create all the directories for the parent
+ FileStatus parentStatus = getFileStatus(parent);
+ LOG.trace("parent key:{} status:{}", key, parentStatus);
+ } catch (FileNotFoundException e) {
+ mkdirs(parent);
+ }
+ // This exception needs to ignored as this means that the file currently
+ // does not exists and a new file can thus be created.
+ }
+
+ OzoneOutputStream ozoneOutputStream =
+ bucket.createKey(key, 0, replicationType, replicationFactor);
+ // We pass null to FSDataOutputStream so it won't count writes that
+ // are being buffered to a file
+ return new FSDataOutputStream(
+ new OzoneFSOutputStream(ozoneOutputStream.getOutputStream()), null);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ final Path parent = path.getParent();
+ if (parent != null) {
+ // expect this to raise an exception if there is no parent
+ if (!getFileStatus(parent).isDirectory()) {
+ throw new FileAlreadyExistsException("Not a directory: " + parent);
+ }
+ }
+ return create(path, permission, flags.contains(CreateFlag.OVERWRITE),
+ bufferSize, replication, blockSize, progress);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcKey;
+ private final String dstKey;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ srcKey = pathToKey(srcPath);
+ dstKey = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", srcKey, dstKey);
+ }
+
+ boolean processKey(String key) throws IOException {
+ String newKeyName = dstKey.concat(key.substring(srcKey.length()));
+ bucket.renameKey(key, newKeyName);
+ return true;
+ }
+ }
+
+ /**
+ * Check whether the source and destination path are valid and then perform
+ * rename from source path to destination path.
+ *
+ * The rename operation is performed by renaming the keys with src as prefix.
+ * For such keys the prefix is changed from src to dst.
+ *
+ * @param src source path for rename
+ * @param dst destination path for rename
+ * @return true if rename operation succeeded or
+ * if the src and dst have the same path and are of the same type
+ * @throws IOException on I/O errors or if the src/dst paths are invalid.
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ if (src.equals(dst)) {
+ return true;
+ }
+
+ LOG.trace("rename() from:{} to:{}", src, dst);
+ if (src.isRoot()) {
+ // Cannot rename root of file system
+ LOG.trace("Cannot rename the root of a filesystem");
+ return false;
+ }
+
+ // Cannot rename a directory to its own subdirectory
+ Path dstParent = dst.getParent();
+ while (dstParent != null && !src.equals(dstParent)) {
+ dstParent = dstParent.getParent();
+ }
+ Preconditions.checkArgument(dstParent == null,
+ "Cannot rename a directory to its own subdirectory");
+ // Check if the source exists
+ FileStatus srcStatus;
+ try {
+ srcStatus = getFileStatus(src);
+ } catch (FileNotFoundException fnfe) {
+ // source doesn't exist, return
+ return false;
+ }
+
+ // Check if the destination exists
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst parent dir exists or not
+ // if the parent exists, the source can still be renamed to dst path
+ dstStatus = getFileStatus(dst.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", src, dst,
+ dst.getParent()));
+ }
+ } else {
+ // if dst exists and source and destination are same,
+ // check both the src and dst are of same type
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory, rename source as subpath of it.
+ // for example rename /source to /dst will lead to /dst/source
+ dst = new Path(dst, src.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ src, dst));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", src, dst));
+ }
+ }
+
+ if (srcStatus.isDirectory()) {
+ if (dst.toString().startsWith(src.toString())) {
+ LOG.trace("Cannot rename a directory to a subdirectory of self");
+ return false;
+ }
+ }
+ RenameIterator iterator = new RenameIterator(src, dst);
+ return iterator.iterate();
+ }
+
+ private class DeleteIterator extends OzoneListingIterator {
+ private boolean recursive;
+ DeleteIterator(Path f, boolean recursive)
+ throws IOException {
+ super(f);
+ this.recursive = recursive;
+ if (getStatus().isDirectory()
+ && !this.recursive
+ && listStatus(f).length != 0) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ }
+ }
+
+ boolean processKey(String key) throws IOException {
+ if (key.equals("")) {
+ LOG.trace("Skipping deleting root directory");
+ return true;
+ } else {
+ LOG.trace("deleting key:" + key);
+ boolean succeed = deleteObject(key);
+ // if recursive delete is requested ignore the return value of
+ // deleteObject and issue deletes for other keys.
+ return recursive || succeed;
+ }
+ }
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive) throws IOException {
+ LOG.trace("delete() path:{} recursive:{}", f, recursive);
+ try {
+ DeleteIterator iterator = new DeleteIterator(f, recursive);
+ return iterator.iterate();
+ } catch (FileNotFoundException e) {
+ LOG.debug("Couldn't delete {} - does not exist", f);
+ return false;
+ }
+ }
+
+ private class ListStatusIterator extends OzoneListingIterator {
+ private List<FileStatus> statuses = new ArrayList<>(LISTING_PAGE_SIZE);
+ private Path f;
+
+ ListStatusIterator(Path f) throws IOException {
+ super(f);
+ this.f = f;
+ }
+
+ boolean processKey(String key) throws IOException {
+ Path keyPath = new Path(OZONE_URI_DELIMITER + key);
+ if (key.equals(getPathKey())) {
+ if (pathIsDirectory()) {
+ return true;
+ } else {
+ statuses.add(getFileStatus(keyPath));
+ return true;
+ }
+ }
+ // left with only subkeys now
+ if (pathToKey(keyPath.getParent()).equals(pathToKey(f))) {
+ // skip keys which are for subdirectories of the directory
+ statuses.add(getFileStatus(keyPath));
+ }
+ return true;
+ }
+
+ FileStatus[] getStatuses() {
+ return statuses.toArray(new FileStatus[statuses.size()]);
+ }
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+ LOG.trace("listStatus() path:{}", f);
+ ListStatusIterator iterator = new ListStatusIterator(f);
+ iterator.iterate();
+ return iterator.getStatuses();
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newDir) {
+ workingDir = newDir;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ /**
+ * Check whether the path is valid and then create directories.
+ * Directory is represented using a key with no value.
+ * All the non-existent parent directories are also created.
+ *
+ * @param path directory path to be created
+ * @return true if directory exists or created successfully.
+ * @throws IOException
+ */
+ private boolean mkdir(Path path) throws IOException {
+ Path fPart = path;
+ Path prevfPart = null;
+ do {
+ LOG.trace("validating path:{}", fPart);
+ try {
+ FileStatus fileStatus = getFileStatus(fPart);
+ if (fileStatus.isDirectory()) {
+ // If path exists and a directory, exit
+ break;
+ } else {
+ // Found a file here, rollback and delete newly created directories
+ LOG.trace("Found a file with same name as directory, path:{}", fPart);
+ if (prevfPart != null) {
+ delete(prevfPart, true);
+ }
+ throw new FileAlreadyExistsException(String.format(
+ "Can't make directory for path '%s', it is a file.", fPart));
+ }
+ } catch (FileNotFoundException fnfe) {
+ LOG.trace("creating directory for fpart:{}", fPart);
+ String key = pathToKey(fPart);
+ String dirKey = addTrailingSlashIfNeeded(key);
+ if (!createDirectory(dirKey)) {
+ // Directory creation failed here,
+ // rollback and delete newly created directories
+ LOG.trace("Directory creation failed, path:{}", fPart);
+ if (prevfPart != null) {
+ delete(prevfPart, true);
+ }
+ return false;
+ }
+ }
+ prevfPart = fPart;
+ fPart = fPart.getParent();
+ } while (fPart != null);
+ return true;
+ }
+
+ @Override
+ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ LOG.trace("mkdir() path:{} ", f);
+ String key = pathToKey(f);
+ if (StringUtils.isEmpty(key)) {
+ return false;
+ }
+ return mkdir(f);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ LOG.trace("getFileStatus() path:{}", f);
+ Path qualifiedPath = f.makeQualified(uri, workingDir);
+ String key = pathToKey(qualifiedPath);
+
+ if (key.length() == 0) {
+ return new FileStatus(0, true, 1, 0,
+ bucket.getCreationTime(), qualifiedPath);
+ }
+
+ // consider this a file and get key status
+ OzoneKey meta = getKeyInfo(key);
+ if (meta == null) {
+ key = addTrailingSlashIfNeeded(key);
+ meta = getKeyInfo(key);
+ }
+
+ if (meta == null) {
+ LOG.trace("File:{} not found", f);
+ throw new FileNotFoundException(f + ": No such file or directory!");
+ } else if (isDirectory(meta)) {
+ return new FileStatus(0, true, 1, 0,
+ meta.getModificationTime(), qualifiedPath);
+ } else {
+ //TODO: Fetch replication count from ratis config
+ return new FileStatus(meta.getDataSize(), false, 1,
+ getDefaultBlockSize(f), meta.getModificationTime(), qualifiedPath);
+ }
+ }
+
+ /**
+ * Helper method to fetch the key metadata info.
+ * @param key key whose metadata information needs to be fetched
+ * @return metadata info of the key
+ */
+ private OzoneKey getKeyInfo(String key) {
+ try {
+ return bucket.getKey(key);
+ } catch (IOException e) {
+ LOG.trace("Key:{} does not exists", key);
+ return null;
+ }
+ }
+
+ /**
+ * Helper method to check if an Ozone key is representing a directory.
+ * @param key key to be checked as a directory
+ * @return true if key is a directory, false otherwise
+ */
+ private boolean isDirectory(OzoneKey key) {
+ LOG.trace("key name:{} size:{}", key.getName(),
+ key.getDataSize());
+ return key.getName().endsWith(OZONE_URI_DELIMITER)
+ && (key.getDataSize() == 0);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ * @param keyName key name to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ private boolean createDirectory(String keyName) {
+ try {
+ LOG.trace("creating dir for key:{}", keyName);
+ bucket.createKey(keyName, 0, replicationType, replicationFactor).close();
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("create key failed for key:{}", keyName, ioe);
+ return false;
+ }
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ * @param keyName key name to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ private boolean deleteObject(String keyName) {
+ LOG.trace("issuing delete for key" + keyName);
+ try {
+ bucket.deleteKey(keyName);
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("delete key failed " + ioe.getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * Turn a path (relative or otherwise) into an Ozone key.
+ *
+ * @param path the path of the file.
+ * @return the key of the object that represents the file.
+ */
+ public String pathToKey(Path path) {
+ Objects.requireNonNull(path, "Path can not be null!");
+ if (!path.isAbsolute()) {
+ path = new Path(workingDir, path);
+ }
+ // removing leading '/' char
+ String key = path.toUri().getPath().substring(1);
+ LOG.trace("path for key:{} is:{}", key, path);
+ return key;
+ }
+
+ /**
+ * Add trailing delimiter to path if it is already not present.
+ *
+ * @param key the ozone Key which needs to be appended
+ * @return delimiter appended key
+ */
+ private String addTrailingSlashIfNeeded(String key) {
+ if (StringUtils.isNotEmpty(key) && !key.endsWith(OZONE_URI_DELIMITER)) {
+ return key + OZONE_URI_DELIMITER;
+ } else {
+ return key;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "OzoneFileSystem{URI=" + uri + ", "
+ + "workingDir=" + workingDir + ", "
+ + "userName=" + userName + ", "
+ + "statistics=" + statistics
+ + "}";
+ }
+
+ private abstract class OzoneListingIterator {
+ private final Path path;
+ private final FileStatus status;
+ private String pathKey;
+ private Iterator<OzoneKey> keyIterator;
+
+ OzoneListingIterator(Path path)
+ throws IOException {
+ this.path = path;
+ this.status = getFileStatus(path);
+ this.pathKey = pathToKey(path);
+ if (status.isDirectory()) {
+ this.pathKey = addTrailingSlashIfNeeded(pathKey);
+ }
+ keyIterator = bucket.listKeys(pathKey);
+ }
+
+ abstract boolean processKey(String key) throws IOException;
+
+ // iterates all the keys in the particular path
+ boolean iterate() throws IOException {
+ LOG.trace("Iterating path {}", path);
+ if (status.isDirectory()) {
+ LOG.trace("Iterating directory:{}", pathKey);
+ while (keyIterator.hasNext()) {
+ OzoneKey key = keyIterator.next();
+ LOG.trace("iterating key:{}", key.getName());
+ if (!processKey(key.getName())) {
+ return false;
+ }
+ }
+ return true;
+ } else {
+ LOG.trace("iterating file:{}", path);
+ return processKey(pathKey);
+ }
+ }
+
+ String getPathKey() {
+ return pathKey;
+ }
+
+ boolean pathIsDirectory() {
+ return status.isDirectory();
+ }
+
+ FileStatus getStatus() {
+ return status;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
new file mode 100644
index 0000000..93e82c3
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Ozone Filesystem.
+ *
+ * Except for the exceptions, it should all be hidden as implementation details.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
new file mode 100644
index 0000000..ad21f28
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.web.handlers.BucketArgs;
+import org.apache.hadoop.ozone.web.handlers.UserArgs;
+import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Test OzoneFSInputStream by reading through multiple interfaces.
+ */
+public class TestOzoneFSInputStream {
+ private static MiniOzoneCluster cluster = null;
+ private static FileSystem fs;
+ private static StorageHandler storageHandler;
+ private static Path filePath = null;
+ private static byte[] data = null;
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true and
+ * OZONE_HANDLER_TYPE_KEY = "distributed"
+ *
+ * @throws IOException
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 10);
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(10)
+ .build();
+ cluster.waitForClusterToBeReady();
+ storageHandler =
+ new ObjectStoreHandler(conf).getStorageHandler();
+
+ // create a volume and a bucket to be used by OzoneFileSystem
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
+ null, null, null, null);
+ VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
+ volumeArgs.setUserName(userName);
+ volumeArgs.setAdminName(adminName);
+ storageHandler.createVolume(volumeArgs);
+ BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
+ storageHandler.createBucket(bucketArgs);
+
+ // Fetch the host and port for File System init
+ DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0)
+ .getDatanodeDetails();
+ int port = datanodeDetails
+ .getPort(DatanodeDetails.Port.Name.REST).getValue();
+ String host = datanodeDetails.getHostName();
+
+ // Set the fs.defaultFS and start the filesystem
+ String uri = String.format("%s://%s.%s/",
+ OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, uri);
+ fs = FileSystem.get(conf);
+ int fileLen = 100 * 1024 * 1024;
+ data = DFSUtil.string2Bytes(RandomStringUtils.randomAlphanumeric(fileLen));
+ filePath = new Path("/" + RandomStringUtils.randomAlphanumeric(5));
+ try (FSDataOutputStream stream = fs.create(filePath)) {
+ stream.write(data);
+ }
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() throws IOException {
+ fs.close();
+ storageHandler.close();
+ cluster.shutdown();
+ }
+
+ @Test
+ public void testO3FSSingleByteRead() throws IOException {
+ FSDataInputStream inputStream = fs.open(filePath);
+ byte[] value = new byte[data.length];
+ int i = 0;
+ while(true) {
+ int val = inputStream.read();
+ if (val == -1) {
+ break;
+ }
+ value[i] = (byte)val;
+ Assert.assertEquals("value mismatch at:" + i, value[i], data[i]);
+ i++;
+ }
+ Assert.assertEquals(i, data.length);
+ Assert.assertTrue(Arrays.equals(value, data));
+ inputStream.close();
+ }
+
+ @Test
+ public void testO3FSMultiByteRead() throws IOException {
+ FSDataInputStream inputStream = fs.open(filePath);
+ byte[] value = new byte[data.length];
+ byte[] tmp = new byte[1* 1024 *1024];
+ int i = 0;
+ while(true) {
+ int val = inputStream.read(tmp);
+ if (val == -1) {
+ break;
+ }
+ System.arraycopy(tmp, 0, value, i * tmp.length, tmp.length);
+ i++;
+ }
+ Assert.assertEquals(i * tmp.length, data.length);
+ Assert.assertTrue(Arrays.equals(value, data));
+ inputStream.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
new file mode 100644
index 0000000..a225702
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.After;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
+import org.apache.hadoop.ozone.web.handlers.BucketArgs;
+import org.apache.hadoop.ozone.web.handlers.UserArgs;
+import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
+import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
+
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test OzoneFileSystem Interfaces.
+ *
+ * This test will test the various interfaces i.e.
+ * create, read, write, getFileStatus
+ */
+@RunWith(Parameterized.class)
+public class TestOzoneFileInterfaces {
+
+ private String rootPath;
+ private String userName;
+
+ /**
+ * Parameter class to set absolute url/defaultFS handling.
+ * <p>
+ * Hadoop file systems could be used in multiple ways: Using the defaultfs
+ * and file path without the schema, or use absolute url-s even with
+ * different defaultFS. This parameter matrix would test both the use cases.
+ */
+ @Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {{false, true}, {true, false}});
+ }
+
+ private boolean setDefaultFs;
+
+ private boolean useAbsolutePath;
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+
+ private static StorageHandler storageHandler;
+
+ public TestOzoneFileInterfaces(boolean setDefaultFs,
+ boolean useAbsolutePath) {
+ this.setDefaultFs = setDefaultFs;
+ this.useAbsolutePath = useAbsolutePath;
+ }
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ storageHandler =
+ new ObjectStoreHandler(conf).getStorageHandler();
+
+ // create a volume and a bucket to be used by OzoneFileSystem
+ userName = "user" + RandomStringUtils.randomNumeric(5);
+ String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
+ null, null, null, null);
+ VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
+ volumeArgs.setUserName(userName);
+ volumeArgs.setAdminName(adminName);
+ storageHandler.createVolume(volumeArgs);
+ BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
+ storageHandler.createBucket(bucketArgs);
+
+ rootPath = String
+ .format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucketName,
+ volumeName);
+ if (setDefaultFs) {
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ fs = FileSystem.get(conf);
+ } else {
+ fs = FileSystem.get(new URI(rootPath + "/test.txt"), conf);
+ }
+ }
+
+ @After
+ public void teardown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.closeQuietly(fs);
+ IOUtils.closeQuietly(storageHandler);
+ }
+
+ @Test
+ public void testFileSystemInit() throws IOException {
+ if (setDefaultFs) {
+ assertTrue(
+ "The initialized file system is not OzoneFileSystem but " +
+ fs.getClass(),
+ fs instanceof OzoneFileSystem);
+ assertEquals(OzoneConsts.OZONE_URI_SCHEME, fs.getUri().getScheme());
+ }
+ }
+
+ @Test
+ public void testOzFsReadWrite() throws IOException {
+ long currentTime = Time.now();
+ int stringLen = 20;
+ String data = RandomStringUtils.randomAlphanumeric(stringLen);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + filePath);
+ try (FSDataOutputStream stream = fs.create(path)) {
+ stream.writeBytes(data);
+ }
+
+ FileStatus status = fs.getFileStatus(path);
+ // The timestamp of the newly created file should always be greater than
+ // the time when the test was started
+ assertTrue("Modification time has not been recorded: " + status,
+ status.getModificationTime() > currentTime);
+
+ try (FSDataInputStream inputStream = fs.open(path)) {
+ byte[] buffer = new byte[stringLen];
+ inputStream.readFully(0, buffer);
+ String out = new String(buffer, 0, buffer.length);
+ assertEquals(data, out);
+ }
+ }
+
+
+ @Test
+ public void testDirectory() throws IOException {
+ String dirPath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + dirPath);
+ assertTrue("Makedirs returned with false for the path " + path,
+ fs.mkdirs(path));
+
+ FileStatus status = fs.getFileStatus(path);
+ assertTrue("The created path is not directory.", status.isDirectory());
+
+ assertEquals(0, status.getLen());
+
+ FileStatus[] statusList = fs.listStatus(createPath("/"));
+ assertEquals(1, statusList.length);
+ assertEquals(status, statusList[0]);
+
+ FileStatus statusRoot = fs.getFileStatus(createPath("/"));
+ assertTrue("Root dir (/) is not a directory.", status.isDirectory());
+ assertEquals(0, status.getLen());
+
+
+ }
+
+ @Test
+ public void testPathToKey() throws Exception {
+ OzoneFileSystem ozoneFs = (OzoneFileSystem) TestOzoneFileInterfaces.fs;
+
+ assertEquals("a/b/1", ozoneFs.pathToKey(new Path("/a/b/1")));
+
+ assertEquals("user/" + getCurrentUser() + "/key1/key2",
+ ozoneFs.pathToKey(new Path("key1/key2")));
+
+ assertEquals("key1/key2",
+ ozoneFs.pathToKey(new Path("o3://test1/key1/key2")));
+ }
+
+ private String getCurrentUser() {
+ try {
+ return UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ return OZONE_DEFAULT_USER;
+ }
+ }
+
+ private Path createPath(String relativePath) {
+ if (useAbsolutePath) {
+ return new Path(
+ rootPath + (relativePath.startsWith("/") ? "" : "/") + relativePath);
+ } else {
+ return new Path(relativePath);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
new file mode 100644
index 0000000..dd54315
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests creating files.
+ */
+public class ITestOzoneContractCreate extends AbstractContractCreateTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
new file mode 100644
index 0000000..f0a3d8d
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests covering deletes.
+ */
+public class ITestOzoneContractDelete extends AbstractContractDeleteTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
new file mode 100644
index 0000000..134a9ad
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+
+/**
+ * Contract test suite covering S3A integration with DistCp.
+ * Uses the block output stream, buffered to disk. This is the
+ * recommended output mechanism for DistCP due to its scalability.
+ */
+public class ITestOzoneContractDistCp extends AbstractContractDistCpTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected OzoneContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
new file mode 100644
index 0000000..98bbb14
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests covering getFileStatus.
+ */
+public class ITestOzoneContractGetFileStatus
+ extends AbstractContractGetFileStatusTest {
+
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+
+ @Override
+ public void teardown() throws Exception {
+ getLog().info("FS details {}", getFileSystem());
+ super.teardown();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return super.createConfiguration();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
new file mode 100644
index 0000000..bc0de5d
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Test dir operations on Ozone.
+ */
+public class ITestOzoneContractMkdir extends AbstractContractMkdirTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
new file mode 100644
index 0000000..0bc57d4
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests opening files.
+ */
+public class ITestOzoneContractOpen extends AbstractContractOpenTest {
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
new file mode 100644
index 0000000..8ce1d1b
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests covering rename.
+ */
+public class ITestOzoneContractRename extends AbstractContractRenameTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HADOOP-15549. Upgrade to
commons-configuration 2.1 regresses task CPU consumption
Posted by xk...@apache.org.
HADOOP-15549. Upgrade to commons-configuration 2.1 regresses task CPU consumption
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59de9679
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59de9679
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59de9679
Branch: refs/heads/HDFS-12943
Commit: 59de9679540f6d0edfb34cf9f88e52b51d94b4f4
Parents: 9f15483
Author: Todd Lipcon <to...@apache.org>
Authored: Thu Jun 21 10:32:52 2018 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Thu Jun 21 10:32:52 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/metrics2/impl/MetricsConfig.java | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59de9679/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java
index 027450c..976f16b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java
@@ -37,10 +37,8 @@ import com.google.common.collect.Maps;
import org.apache.commons.configuration2.Configuration;
import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.configuration2.SubsetConfiguration;
-import org.apache.commons.configuration2.builder.fluent.Configurations;
-import org.apache.commons.configuration2.builder.fluent.Parameters;
-import org.apache.commons.configuration2.convert.DefaultListDelimiterHandler;
import org.apache.commons.configuration2.ex.ConfigurationException;
+import org.apache.commons.configuration2.io.FileHandler;
import org.apache.hadoop.metrics2.MetricsFilter;
import org.apache.hadoop.metrics2.MetricsPlugin;
import org.apache.hadoop.metrics2.filter.GlobFilter;
@@ -112,12 +110,11 @@ class MetricsConfig extends SubsetConfiguration {
static MetricsConfig loadFirst(String prefix, String... fileNames) {
for (String fname : fileNames) {
try {
- Configuration cf = new Configurations().propertiesBuilder(fname)
- .configure(new Parameters().properties()
- .setFileName(fname)
- .setListDelimiterHandler(new DefaultListDelimiterHandler(',')))
- .getConfiguration()
- .interpolatedConfiguration();
+ PropertiesConfiguration pcf = new PropertiesConfiguration();
+ FileHandler fh = new FileHandler(pcf);
+ fh.setFileName(fname);
+ fh.load();
+ Configuration cf = pcf.interpolatedConfiguration();
LOG.info("Loaded properties from {}", fname);
if (LOG.isDebugEnabled()) {
LOG.debug("Properties: {}", toString(cf));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-8441. Typo in CSQueueUtils
local variable names: queueGuranteedResource. Contributed by Szilard Nemeth.
Posted by xk...@apache.org.
YARN-8441. Typo in CSQueueUtils local variable names: queueGuranteedResource. Contributed by Szilard Nemeth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46f90581
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46f90581
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46f90581
Branch: refs/heads/HDFS-12943
Commit: 46f90581641feec37e285964df983d221bee5e1d
Parents: 388fafa
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Wed Jun 20 11:58:18 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Wed Jun 20 11:58:18 2018 -0700
----------------------------------------------------------------------
.../scheduler/capacity/CSQueueUtils.java | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f90581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 0dfce83..b5edbf7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -184,7 +184,7 @@ public class CSQueueUtils {
if (Resources.greaterThan(rc, totalPartitionResource,
totalPartitionResource, Resources.none())) {
- Resource queueGuranteedResource = childQueue
+ Resource queueGuaranteedResource = childQueue
.getEffectiveCapacity(nodePartition);
//TODO : Modify below code to support Absolute Resource configurations
@@ -204,14 +204,14 @@ public class CSQueueUtils {
QueueCapacities leafQueueTemplateCapacities = parentQueue
.getLeafQueueTemplate()
.getQueueCapacities();
- queueGuranteedResource = Resources.multiply(totalPartitionResource,
+ queueGuaranteedResource = Resources.multiply(totalPartitionResource,
leafQueueTemplateCapacities.getAbsoluteCapacity
(nodePartition));
}
// make queueGuranteed >= minimum_allocation to avoid divided by 0.
- queueGuranteedResource =
- Resources.max(rc, totalPartitionResource, queueGuranteedResource,
+ queueGuaranteedResource =
+ Resources.max(rc, totalPartitionResource, queueGuaranteedResource,
minimumAllocation);
Resource usedResource = queueResourceUsage.getUsed(nodePartition);
@@ -220,12 +220,12 @@ public class CSQueueUtils {
totalPartitionResource);
usedCapacity =
Resources.divide(rc, totalPartitionResource, usedResource,
- queueGuranteedResource);
+ queueGuaranteedResource);
Resource resResource = queueResourceUsage.getReserved(nodePartition);
reservedCapacity =
Resources.divide(rc, totalPartitionResource, resResource,
- queueGuranteedResource);
+ queueGuaranteedResource);
absoluteReservedCapacity =
Resources.divide(rc, totalPartitionResource, resResource,
totalPartitionResource);
@@ -258,16 +258,16 @@ public class CSQueueUtils {
for (String partition : nodeLabels) {
// Calculate guaranteed resource for a label in a queue by below logic.
// (total label resource) * (absolute capacity of label in that queue)
- Resource queueGuranteedResource = queue.getEffectiveCapacity(partition);
+ Resource queueGuaranteedResource = queue.getEffectiveCapacity(partition);
// Available resource in queue for a specific label will be calculated as
// {(guaranteed resource for a label in a queue) -
// (resource usage of that label in the queue)}
// Finally accumulate this available resource to get total.
Resource available = (Resources.greaterThan(rc, cluster,
- queueGuranteedResource,
+ queueGuaranteedResource,
queue.getQueueResourceUsage().getUsed(partition))) ? Resources
- .componentwiseMax(Resources.subtractFrom(queueGuranteedResource,
+ .componentwiseMax(Resources.subtractFrom(queueGuaranteedResource,
queue.getQueueResourceUsage().getUsed(partition)), Resources
.none()) : Resources.none();
Resources.addTo(totalAvailableResource, available);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HADOOP-15551. Avoid use of
Arrays.stream in Configuration.addTags
Posted by xk...@apache.org.
HADOOP-15551. Avoid use of Arrays.stream in Configuration.addTags
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43541a18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43541a18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43541a18
Branch: refs/heads/HDFS-12943
Commit: 43541a18907d2303b708ae27a9a2cb5df891da4f
Parents: 32f867a
Author: Todd Lipcon <to...@apache.org>
Authored: Wed Jun 20 12:38:59 2018 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Wed Jun 20 16:43:10 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/conf/Configuration.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43541a18/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 19bd5da..b1125e5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -3189,25 +3189,25 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
if (prop.containsKey(CommonConfigurationKeys.HADOOP_TAGS_SYSTEM)) {
String systemTags = prop.getProperty(CommonConfigurationKeys
.HADOOP_TAGS_SYSTEM);
- Arrays.stream(systemTags.split(",")).forEach(tag -> TAGS.add(tag));
+ TAGS.addAll(Arrays.asList(systemTags.split(",")));
}
// Get all custom tags
if (prop.containsKey(CommonConfigurationKeys.HADOOP_TAGS_CUSTOM)) {
String customTags = prop.getProperty(CommonConfigurationKeys
.HADOOP_TAGS_CUSTOM);
- Arrays.stream(customTags.split(",")).forEach(tag -> TAGS.add(tag));
+ TAGS.addAll(Arrays.asList(customTags.split(",")));
}
if (prop.containsKey(CommonConfigurationKeys.HADOOP_SYSTEM_TAGS)) {
String systemTags = prop.getProperty(CommonConfigurationKeys
.HADOOP_SYSTEM_TAGS);
- Arrays.stream(systemTags.split(",")).forEach(tag -> TAGS.add(tag));
+ TAGS.addAll(Arrays.asList(systemTags.split(",")));
}
// Get all custom tags
if (prop.containsKey(CommonConfigurationKeys.HADOOP_CUSTOM_TAGS)) {
String customTags = prop.getProperty(CommonConfigurationKeys
.HADOOP_CUSTOM_TAGS);
- Arrays.stream(customTags.split(",")).forEach(tag -> TAGS.add(tag));
+ TAGS.addAll(Arrays.asList(customTags.split(",")));
}
} catch (Exception ex) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: YARN-8444: NodeResourceMonitor
crashes on bad swapFree value. Contributed by Jim Brennan.
Posted by xk...@apache.org.
YARN-8444: NodeResourceMonitor crashes on bad swapFree value. Contributed by Jim Brennan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64321286
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64321286
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64321286
Branch: refs/heads/HDFS-12943
Commit: 6432128622d64f3f9dd638b9c254c77cdf5408aa
Parents: 30728ac
Author: Eric E Payne <er...@oath.com>
Authored: Fri Jun 22 17:15:29 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Fri Jun 22 17:15:29 2018 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/util/SysInfoLinux.java | 19 ++++++-
.../apache/hadoop/util/TestSysInfoLinux.java | 60 ++++++++++++++++++++
2 files changed, 77 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64321286/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
index 7fd1990..2c2aca3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
@@ -217,6 +217,21 @@ public class SysInfoLinux extends SysInfo {
}
/**
+ *
+ * Wrapper for Long.parseLong() that returns zero if the value is
+ * invalid. Under some circumstances, swapFree in /proc/meminfo can
+ * go negative, reported as a very large decimal value.
+ */
+ private long safeParseLong(String strVal) {
+ long parsedVal;
+ try {
+ parsedVal = Long.parseLong(strVal);
+ } catch (NumberFormatException nfe) {
+ parsedVal = 0;
+ }
+ return parsedVal;
+ }
+ /**
* Read /proc/meminfo, parse and compute memory information.
* @param readAgain if false, read only on the first time
*/
@@ -252,9 +267,9 @@ public class SysInfoLinux extends SysInfo {
} else if (mat.group(1).equals(SWAPTOTAL_STRING)) {
swapSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(MEMFREE_STRING)) {
- ramSizeFree = Long.parseLong(mat.group(2));
+ ramSizeFree = safeParseLong(mat.group(2));
} else if (mat.group(1).equals(SWAPFREE_STRING)) {
- swapSizeFree = Long.parseLong(mat.group(2));
+ swapSizeFree = safeParseLong(mat.group(2));
} else if (mat.group(1).equals(INACTIVE_STRING)) {
inactiveSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(INACTIVEFILE_STRING)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64321286/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
index a646a41..0ae5d3c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
@@ -161,6 +161,36 @@ public class TestSysInfoLinux {
"DirectMap2M: 2027520 kB\n" +
"DirectMap1G: 132120576 kB\n";
+ static final String MEMINFO_FORMAT3 =
+ "MemTotal: %d kB\n" +
+ "MemFree: %s kB\n" +
+ "Buffers: 138244 kB\n" +
+ "Cached: 947780 kB\n" +
+ "SwapCached: 142880 kB\n" +
+ "Active: 3229888 kB\n" +
+ "Inactive: %d kB\n" +
+ "SwapTotal: %d kB\n" +
+ "SwapFree: %s kB\n" +
+ "Dirty: 122012 kB\n" +
+ "Writeback: 0 kB\n" +
+ "AnonPages: 2710792 kB\n" +
+ "Mapped: 24740 kB\n" +
+ "Slab: 132528 kB\n" +
+ "SReclaimable: 105096 kB\n" +
+ "SUnreclaim: 27432 kB\n" +
+ "PageTables: 11448 kB\n" +
+ "NFS_Unstable: 0 kB\n" +
+ "Bounce: 0 kB\n" +
+ "CommitLimit: 4125904 kB\n" +
+ "Committed_AS: 4143556 kB\n" +
+ "VmallocTotal: 34359738367 kB\n" +
+ "VmallocUsed: 1632 kB\n" +
+ "VmallocChunk: 34359736375 kB\n" +
+ "HugePages_Total: %d\n" +
+ "HugePages_Free: 0\n" +
+ "HugePages_Rsvd: 0\n" +
+ "Hugepagesize: 2048 kB";
+
static final String CPUINFO_FORMAT =
"processor : %s\n" +
"vendor_id : AuthenticAMD\n" +
@@ -384,6 +414,36 @@ public class TestSysInfoLinux {
(nrHugePages * 2048) + swapTotal));
}
+ /**
+ * Test parsing /proc/meminfo
+ * @throws IOException
+ */
+ @Test
+ public void parsingProcMemFileWithBadValues() throws IOException {
+ long memTotal = 4058864L;
+ long memFree = 0L; // bad value should return 0
+ long inactive = 567732L;
+ long swapTotal = 2096472L;
+ long swapFree = 0L; // bad value should return 0
+ int nrHugePages = 10;
+ String badFreeValue = "18446744073709551596";
+ File tempFile = new File(FAKE_MEMFILE);
+ tempFile.deleteOnExit();
+ FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
+ fWriter.write(String.format(MEMINFO_FORMAT3,
+ memTotal, badFreeValue, inactive, swapTotal, badFreeValue, nrHugePages));
+
+ fWriter.close();
+ assertEquals(plugin.getAvailablePhysicalMemorySize(),
+ 1024L * (memFree + inactive));
+ assertEquals(plugin.getAvailableVirtualMemorySize(),
+ 1024L * (memFree + inactive + swapFree));
+ assertEquals(plugin.getPhysicalMemorySize(),
+ 1024L * (memTotal - (nrHugePages * 2048)));
+ assertEquals(plugin.getVirtualMemorySize(),
+ 1024L * (memTotal - (nrHugePages * 2048) + swapTotal));
+ }
+
@Test
public void testCoreCounts() throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-8412. Move
ResourceRequest.clone logic everywhere into a proper API. Contributed by
Botong Huang.
Posted by xk...@apache.org.
YARN-8412. Move ResourceRequest.clone logic everywhere into a proper API. Contributed by Botong Huang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99948565
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99948565
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99948565
Branch: refs/heads/HDFS-12943
Commit: 99948565cb5d5706241d7a8fc591e1617c499e03
Parents: 59de967
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Jun 21 18:24:10 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Jun 21 18:24:10 2018 -0700
----------------------------------------------------------------------
.../yarn/api/records/ResourceRequest.java | 20 ++++++++++++++++++++
.../yarn/client/api/impl/AMRMClientImpl.java | 11 +----------
.../api/impl/TestAMRMClientOnRMRestart.java | 6 +-----
.../hadoop/yarn/server/AMRMClientRelayer.java | 8 +-------
.../LocalityMulticastAMRMProxyPolicy.java | 10 +---------
.../server/scheduler/ResourceRequestSet.java | 14 ++------------
.../hadoop/yarn/server/utils/BuilderUtils.java | 12 ------------
.../LocalityAppPlacementAllocator.java | 10 ++--------
.../server/resourcemanager/Application.java | 10 ++++------
.../server/resourcemanager/TestAppManager.java | 18 ++----------------
10 files changed, 34 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99948565/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index eea81fe..a863910 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -102,6 +102,26 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
.build();
}
+ /**
+ * Clone a ResourceRequest object (shallow copy). Please keep it loaded with
+ * all (new) fields
+ *
+ * @param rr the object to copy from
+ * @return the copied object
+ */
+ @Public
+ @Evolving
+ public static ResourceRequest clone(ResourceRequest rr) {
+ // Please keep it loaded with all (new) fields
+ return ResourceRequest.newBuilder().priority(rr.getPriority())
+ .resourceName(rr.getResourceName()).capability(rr.getCapability())
+ .numContainers(rr.getNumContainers())
+ .relaxLocality(rr.getRelaxLocality())
+ .nodeLabelExpression(rr.getNodeLabelExpression())
+ .executionTypeRequest(rr.getExecutionTypeRequest())
+ .allocationRequestId(rr.getAllocationRequestId()).build();
+ }
+
@Public
@Unstable
public static ResourceRequestBuilder newBuilder() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99948565/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
index 36c3cf1..7265d24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
@@ -451,16 +451,7 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
for(ResourceRequest r : ask) {
// create a copy of ResourceRequest as we might change it while the
// RPC layer is using it to send info across
- ResourceRequest rr =
- ResourceRequest.newBuilder().priority(r.getPriority())
- .resourceName(r.getResourceName()).capability(r.getCapability())
- .numContainers(r.getNumContainers())
- .relaxLocality(r.getRelaxLocality())
- .nodeLabelExpression(r.getNodeLabelExpression())
- .executionTypeRequest(r.getExecutionTypeRequest())
- .allocationRequestId(r.getAllocationRequestId())
- .build();
- askList.add(rr);
+ askList.add(ResourceRequest.clone(r));
}
return askList;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99948565/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
index 11d703d..5104866 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
@@ -570,11 +570,7 @@ public class TestAMRMClientOnRMRestart {
ContainerUpdates updateRequests) {
List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
for (ResourceRequest req : ask) {
- ResourceRequest reqCopy =
- ResourceRequest.newInstance(req.getPriority(),
- req.getResourceName(), req.getCapability(),
- req.getNumContainers(), req.getRelaxLocality());
- askCopy.add(reqCopy);
+ askCopy.add(ResourceRequest.clone(req));
}
lastAsk = ask;
lastRelease = release;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99948565/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
index c216ace..e8a7f64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
@@ -220,13 +220,7 @@ public class AMRMClientRelayer extends AbstractService
for (ResourceRequest r : ask) {
// create a copy of ResourceRequest as we might change it while the
// RPC layer is using it to send info across
- askList.add(ResourceRequest.newBuilder().priority(r.getPriority())
- .resourceName(r.getResourceName()).capability(r.getCapability())
- .numContainers(r.getNumContainers())
- .relaxLocality(r.getRelaxLocality())
- .nodeLabelExpression(r.getNodeLabelExpression())
- .executionTypeRequest(r.getExecutionTypeRequest())
- .allocationRequestId(r.getAllocationRequestId()).build());
+ askList.add(ResourceRequest.clone(r));
}
allocateRequest = AllocateRequest.newBuilder()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99948565/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
index d303d6f..1481f34 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
@@ -361,15 +361,7 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
for (SubClusterId targetId : targetSCs) {
// if the calculated request is non-empty add it to the answer
if (containerNums.get(i) > 0) {
- ResourceRequest out =
- ResourceRequest.newInstance(originalResourceRequest.getPriority(),
- originalResourceRequest.getResourceName(),
- originalResourceRequest.getCapability(),
- originalResourceRequest.getNumContainers(),
- originalResourceRequest.getRelaxLocality(),
- originalResourceRequest.getNodeLabelExpression(),
- originalResourceRequest.getExecutionTypeRequest());
- out.setAllocationRequestId(allocationId);
+ ResourceRequest out = ResourceRequest.clone(originalResourceRequest);
out.setNumContainers(containerNums.get(i));
if (ResourceRequest.isAnyLocation(out.getResourceName())) {
allocationBookkeeper.addAnyRR(targetId, out);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99948565/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSet.java
index b1e6b6e..cf24bbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/ResourceRequestSet.java
@@ -165,7 +165,7 @@ public class ResourceRequestSet {
// the same numContainers value
Map<String, ResourceRequest> newAsks = new HashMap<>();
for (ResourceRequest rr : this.asks.values()) {
- ResourceRequest clone = cloneResourceRequest(rr);
+ ResourceRequest clone = ResourceRequest.clone(rr);
clone.setNumContainers(newValue);
newAsks.put(clone.getResourceName(), clone);
}
@@ -176,22 +176,12 @@ public class ResourceRequestSet {
throw new YarnException(
"No ANY RR found in requestSet with numContainers=" + oldValue);
}
- ResourceRequest clone = cloneResourceRequest(rr);
+ ResourceRequest clone = ResourceRequest.clone(rr);
clone.setNumContainers(newValue);
this.asks.put(ResourceRequest.ANY, clone);
}
}
- private ResourceRequest cloneResourceRequest(ResourceRequest rr) {
- return ResourceRequest.newBuilder().priority(rr.getPriority())
- .resourceName(rr.getResourceName()).capability(rr.getCapability())
- .numContainers(rr.getNumContainers())
- .relaxLocality(rr.getRelaxLocality())
- .nodeLabelExpression(rr.getNodeLabelExpression())
- .executionTypeRequest(rr.getExecutionTypeRequest())
- .allocationRequestId(rr.getAllocationRequestId()).build();
- }
-
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99948565/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index e06b55e..b6145c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -375,18 +375,6 @@ public class BuilderUtils {
return request;
}
- public static ResourceRequest newResourceRequest(ResourceRequest r) {
- ResourceRequest request = recordFactory
- .newRecordInstance(ResourceRequest.class);
- request.setPriority(r.getPriority());
- request.setResourceName(r.getResourceName());
- request.setCapability(r.getCapability());
- request.setNumContainers(r.getNumContainers());
- request.setNodeLabelExpression(r.getNodeLabelExpression());
- request.setExecutionTypeRequest(r.getExecutionTypeRequest());
- return request;
- }
-
public static ApplicationReport newApplicationReport(
ApplicationId applicationId, ApplicationAttemptId applicationAttemptId,
String user, String queue, String name, String host, int rpcPort,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99948565/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
index a0358b4..e1239a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
@@ -251,14 +251,8 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
}
public ResourceRequest cloneResourceRequest(ResourceRequest request) {
- ResourceRequest newRequest = ResourceRequest.newBuilder()
- .priority(request.getPriority())
- .allocationRequestId(request.getAllocationRequestId())
- .resourceName(request.getResourceName())
- .capability(request.getCapability())
- .numContainers(1)
- .relaxLocality(request.getRelaxLocality())
- .nodeLabelExpression(request.getNodeLabelExpression()).build();
+ ResourceRequest newRequest = ResourceRequest.clone(request);
+ newRequest.setNumContainers(1);
return newRequest;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99948565/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
index 7d1140d..9178009 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
@@ -305,9 +305,8 @@ public class Application {
// Note this down for next interaction with ResourceManager
ask.remove(request);
- ask.add(
- org.apache.hadoop.yarn.server.utils.BuilderUtils.newResourceRequest(
- request)); // clone to ensure the RM doesn't manipulate the same obj
+ // clone to ensure the RM doesn't manipulate the same obj
+ ask.add(ResourceRequest.clone(request));
if (LOG.isDebugEnabled()) {
LOG.debug("addResourceRequest: applicationId=" + applicationId.getId()
@@ -462,9 +461,8 @@ public class Application {
// Note this for next interaction with ResourceManager
ask.remove(request);
- ask.add(
- org.apache.hadoop.yarn.server.utils.BuilderUtils.newResourceRequest(
- request)); // clone to ensure the RM doesn't manipulate the same obj
+ // clone to ensure the RM doesn't manipulate the same obj
+ ask.add(ResourceRequest.clone(request));
if(LOG.isDebugEnabled()) {
LOG.debug("updateResourceRequest:" + " application=" + applicationId
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99948565/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index e79ba08..6a6f9cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -733,7 +733,7 @@ public class TestAppManager{
ResourceRequest.newInstance(Priority.newInstance(0),
ResourceRequest.ANY, Resources.createResource(1025), 1, true);
req.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
- asContext.setAMContainerResourceRequest(cloneResourceRequest(req));
+ asContext.setAMContainerResourceRequest(ResourceRequest.clone(req));
// getAMContainerResourceRequests uses a singleton list of
// getAMContainerResourceRequest
Assert.assertEquals(req, asContext.getAMContainerResourceRequest());
@@ -1099,25 +1099,11 @@ public class TestAppManager{
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
}
- private static ResourceRequest cloneResourceRequest(ResourceRequest req) {
- return ResourceRequest.newInstance(
- Priority.newInstance(req.getPriority().getPriority()),
- new String(req.getResourceName()),
- Resource.newInstance(req.getCapability().getMemorySize(),
- req.getCapability().getVirtualCores()),
- req.getNumContainers(),
- req.getRelaxLocality(),
- req.getNodeLabelExpression() != null
- ? new String(req.getNodeLabelExpression()) : null,
- ExecutionTypeRequest.newInstance(
- req.getExecutionTypeRequest().getExecutionType()));
- }
-
private static List<ResourceRequest> cloneResourceRequests(
List<ResourceRequest> reqs) {
List<ResourceRequest> cloneReqs = new ArrayList<>();
for (ResourceRequest req : reqs) {
- cloneReqs.add(cloneResourceRequest(req));
+ cloneReqs.add(ResourceRequest.clone(req));
}
return cloneReqs;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDDS-141. Remove PipeLine Class
from SCM and move the data field in the Pipeline to ContainerInfo.
Contributed by Shashikant Banerjee.
Posted by xk...@apache.org.
HDDS-141. Remove PipeLine Class from SCM and move the data field in the Pipeline to ContainerInfo. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a4632d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a4632d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a4632d3
Branch: refs/heads/HDFS-12943
Commit: 2a4632d3d7b82980d10cc90cdfc52afd866cebb8
Parents: f347446
Author: Mukul Kumar Singh <ms...@apache.org>
Authored: Sun Jun 17 23:48:33 2018 -0700
Committer: Mukul Kumar Singh <ms...@apache.org>
Committed: Sun Jun 17 23:48:49 2018 -0700
----------------------------------------------------------------------
.../container/common/helpers/ContainerInfo.java | 32 +++++
.../scm/container/common/helpers/Pipeline.java | 142 +++++++++++--------
.../common/helpers/PipelineChannel.java | 124 ----------------
hadoop-hdds/common/src/main/proto/hdds.proto | 8 +-
.../scm/container/closer/ContainerCloser.java | 6 +-
.../hdds/scm/pipelines/PipelineManager.java | 67 +++++----
.../hdds/scm/pipelines/PipelineSelector.java | 11 +-
.../scm/pipelines/ratis/RatisManagerImpl.java | 13 +-
.../standalone/StandaloneManagerImpl.java | 8 +-
.../hdds/scm/block/TestDeletedBlockLog.java | 8 +-
.../hadoop/ozone/TestMiniOzoneCluster.java | 8 +-
.../ozone/container/ContainerTestHelper.java | 19 +--
.../genesis/BenchMarkContainerStateMap.java | 11 +-
.../genesis/BenchMarkDatanodeDispatcher.java | 6 +-
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 4 +-
15 files changed, 191 insertions(+), 276 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
index 2c38d45..ee05c87 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdds.scm.container.common.helpers;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
@@ -30,6 +31,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.util.Time;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Comparator;
import static java.lang.Math.max;
@@ -63,6 +65,13 @@ public class ContainerInfo
private String owner;
private long containerID;
private long deleteTransactionId;
+ /**
+ * Allows you to maintain private data on ContainerInfo. This is not
+ * serialized via protobuf, just allows us to maintain some private data.
+ */
+ @JsonIgnore
+ private byte[] data;
+
ContainerInfo(
long containerID,
HddsProtos.LifeCycleState state,
@@ -296,6 +305,29 @@ public class ContainerInfo
}
/**
+ * Returns private data that is set on this containerInfo.
+ *
+ * @return blob, the user can interpret it any way they like.
+ */
+ public byte[] getData() {
+ if (this.data != null) {
+ return Arrays.copyOf(this.data, this.data.length);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Set private data on ContainerInfo object.
+ *
+ * @param data -- private data.
+ */
+ public void setData(byte[] data) {
+ if (data != null) {
+ this.data = Arrays.copyOf(data, data.length);
+ }
+ }
+ /**
* Builder class for ContainerInfo.
*/
public static class Builder {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
index 8740838..c5794f4 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
@@ -27,14 +27,14 @@ import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.ser.FilterProvider;
import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
import java.util.List;
/**
@@ -46,7 +46,7 @@ public class Pipeline {
static {
ObjectMapper mapper = new ObjectMapper();
- String[] ignorableFieldNames = {"data"};
+ String[] ignorableFieldNames = {"leaderID", "datanodes"};
FilterProvider filters = new SimpleFilterProvider()
.addFilter(PIPELINE_INFO, SimpleBeanPropertyFilter
.serializeAllExcept(ignorableFieldNames));
@@ -57,38 +57,66 @@ public class Pipeline {
WRITER = mapper.writer(filters);
}
- private PipelineChannel pipelineChannel;
- /**
- * Allows you to maintain private data on pipelines. This is not serialized
- * via protobuf, just allows us to maintain some private data.
- */
@JsonIgnore
- private byte[] data;
+ private String leaderID;
+ @JsonIgnore
+ private Map<String, DatanodeDetails> datanodes;
+ private HddsProtos.LifeCycleState lifeCycleState;
+ private HddsProtos.ReplicationType type;
+ private HddsProtos.ReplicationFactor factor;
+ private String name;
+ // TODO: change to long based id
+ //private long id;
+
/**
* Constructs a new pipeline data structure.
*
- * @param pipelineChannel - transport information for this container
+ * @param leaderID - Leader datanode id
+ * @param lifeCycleState - Pipeline State
+ * @param replicationType - Replication protocol
+ * @param replicationFactor - replication count on datanodes
+ * @param name - pipelineName
*/
- public Pipeline(PipelineChannel pipelineChannel) {
- this.pipelineChannel = pipelineChannel;
- data = null;
+ public Pipeline(String leaderID, HddsProtos.LifeCycleState lifeCycleState,
+ HddsProtos.ReplicationType replicationType,
+ HddsProtos.ReplicationFactor replicationFactor, String name) {
+ this.leaderID = leaderID;
+ this.lifeCycleState = lifeCycleState;
+ this.type = replicationType;
+ this.factor = replicationFactor;
+ this.name = name;
+ datanodes = new TreeMap<>();
}
/**
* Gets pipeline object from protobuf.
*
- * @param pipeline - ProtoBuf definition for the pipeline.
+ * @param pipelineProto - ProtoBuf definition for the pipeline.
* @return Pipeline Object
*/
- public static Pipeline getFromProtoBuf(HddsProtos.Pipeline pipeline) {
- Preconditions.checkNotNull(pipeline);
- PipelineChannel pipelineChannel =
- PipelineChannel.getFromProtoBuf(pipeline.getPipelineChannel());
- return new Pipeline(pipelineChannel);
+ public static Pipeline getFromProtoBuf(
+ HddsProtos.Pipeline pipelineProto) {
+ Preconditions.checkNotNull(pipelineProto);
+ Pipeline pipeline =
+ new Pipeline(pipelineProto.getLeaderID(),
+ pipelineProto.getState(),
+ pipelineProto.getType(),
+ pipelineProto.getFactor(),
+ pipelineProto.getName());
+
+ for (HddsProtos.DatanodeDetailsProto dataID :
+ pipelineProto.getMembersList()) {
+ pipeline.addMember(DatanodeDetails.getFromProtoBuf(dataID));
+ }
+ return pipeline;
}
+ /**
+ * returns the replication count.
+ * @return Replication Factor
+ */
public HddsProtos.ReplicationFactor getFactor() {
- return pipelineChannel.getFactor();
+ return factor;
}
/**
@@ -98,27 +126,42 @@ public class Pipeline {
*/
@JsonIgnore
public DatanodeDetails getLeader() {
- return pipelineChannel.getDatanodes().get(pipelineChannel.getLeaderID());
+ return getDatanodes().get(leaderID);
}
+ public void addMember(DatanodeDetails datanodeDetails) {
+ datanodes.put(datanodeDetails.getUuid().toString(),
+ datanodeDetails);
+ }
+
+ public Map<String, DatanodeDetails> getDatanodes() {
+ return datanodes;
+ }
/**
* Returns the leader host.
*
* @return First Machine.
*/
public String getLeaderHost() {
- return pipelineChannel.getDatanodes()
- .get(pipelineChannel.getLeaderID()).getHostName();
+ return getDatanodes()
+ .get(leaderID).getHostName();
}
/**
+ *
+ * @return lead
+ */
+ public String getLeaderID() {
+ return leaderID;
+ }
+ /**
* Returns all machines that make up this pipeline.
*
* @return List of Machines.
*/
@JsonIgnore
public List<DatanodeDetails> getMachines() {
- return new ArrayList<>(pipelineChannel.getDatanodes().values());
+ return new ArrayList<>(getDatanodes().values());
}
/**
@@ -128,7 +171,7 @@ public class Pipeline {
*/
public List<String> getDatanodeHosts() {
List<String> dataHosts = new ArrayList<>();
- for (DatanodeDetails id : pipelineChannel.getDatanodes().values()) {
+ for (DatanodeDetails id :getDatanodes().values()) {
dataHosts.add(id.getHostName());
}
return dataHosts;
@@ -143,37 +186,22 @@ public class Pipeline {
public HddsProtos.Pipeline getProtobufMessage() {
HddsProtos.Pipeline.Builder builder =
HddsProtos.Pipeline.newBuilder();
- builder.setPipelineChannel(this.pipelineChannel.getProtobufMessage());
- return builder.build();
- }
-
- /**
- * Returns private data that is set on this pipeline.
- *
- * @return blob, the user can interpret it any way they like.
- */
- public byte[] getData() {
- if (this.data != null) {
- return Arrays.copyOf(this.data, this.data.length);
- } else {
- return null;
+ for (DatanodeDetails datanode : datanodes.values()) {
+ builder.addMembers(datanode.getProtoBufMessage());
}
- }
+ builder.setLeaderID(leaderID);
- @VisibleForTesting
- public PipelineChannel getPipelineChannel() {
- return pipelineChannel;
- }
+ if (this.getLifeCycleState() != null) {
+ builder.setState(this.getLifeCycleState());
+ }
+ if (this.getType() != null) {
+ builder.setType(this.getType());
+ }
- /**
- * Set private data on pipeline.
- *
- * @param data -- private data.
- */
- public void setData(byte[] data) {
- if (data != null) {
- this.data = Arrays.copyOf(data, data.length);
+ if (this.getFactor() != null) {
+ builder.setFactor(this.getFactor());
}
+ return builder.build();
}
/**
@@ -182,7 +210,7 @@ public class Pipeline {
* @return - LifeCycleStates.
*/
public HddsProtos.LifeCycleState getLifeCycleState() {
- return pipelineChannel.getLifeCycleState();
+ return lifeCycleState;
}
/**
@@ -191,7 +219,7 @@ public class Pipeline {
* @return - Name of the pipeline
*/
public String getPipelineName() {
- return pipelineChannel.getName();
+ return name;
}
/**
@@ -200,16 +228,16 @@ public class Pipeline {
* @return type - Standalone, Ratis, Chained.
*/
public HddsProtos.ReplicationType getType() {
- return pipelineChannel.getType();
+ return type;
}
@Override
public String toString() {
final StringBuilder b = new StringBuilder(getClass().getSimpleName())
.append("[");
- pipelineChannel.getDatanodes().keySet().stream()
+ getDatanodes().keySet().stream()
.forEach(id -> b.
- append(id.endsWith(pipelineChannel.getLeaderID()) ? "*" + id : id));
+ append(id.endsWith(getLeaderID()) ? "*" + id : id));
b.append(" name:").append(getPipelineName());
if (getType() != null) {
b.append(" type:").append(getType().toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineChannel.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineChannel.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineChannel.java
deleted file mode 100644
index 655751d..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineChannel.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.container.common.helpers;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * PipelineChannel information for a {@link Pipeline}.
- */
-public class PipelineChannel {
- @JsonIgnore
- private String leaderID;
- @JsonIgnore
- private Map<String, DatanodeDetails> datanodes;
- private LifeCycleState lifeCycleState;
- private ReplicationType type;
- private ReplicationFactor factor;
- private String name;
- // TODO: change to long based id
- //private long id;
-
- public PipelineChannel(String leaderID, LifeCycleState lifeCycleState,
- ReplicationType replicationType, ReplicationFactor replicationFactor,
- String name) {
- this.leaderID = leaderID;
- this.lifeCycleState = lifeCycleState;
- this.type = replicationType;
- this.factor = replicationFactor;
- this.name = name;
- datanodes = new TreeMap<>();
- }
-
- public String getLeaderID() {
- return leaderID;
- }
-
- public Map<String, DatanodeDetails> getDatanodes() {
- return datanodes;
- }
-
- public LifeCycleState getLifeCycleState() {
- return lifeCycleState;
- }
-
- public ReplicationType getType() {
- return type;
- }
-
- public ReplicationFactor getFactor() {
- return factor;
- }
-
- public String getName() {
- return name;
- }
-
- public void addMember(DatanodeDetails datanodeDetails) {
- datanodes.put(datanodeDetails.getUuid().toString(),
- datanodeDetails);
- }
-
- @JsonIgnore
- public HddsProtos.PipelineChannel getProtobufMessage() {
- HddsProtos.PipelineChannel.Builder builder =
- HddsProtos.PipelineChannel.newBuilder();
- for (DatanodeDetails datanode : datanodes.values()) {
- builder.addMembers(datanode.getProtoBufMessage());
- }
- builder.setLeaderID(leaderID);
-
- if (this.getLifeCycleState() != null) {
- builder.setState(this.getLifeCycleState());
- }
- if (this.getType() != null) {
- builder.setType(this.getType());
- }
-
- if (this.getFactor() != null) {
- builder.setFactor(this.getFactor());
- }
- return builder.build();
- }
-
- public static PipelineChannel getFromProtoBuf(
- HddsProtos.PipelineChannel transportProtos) {
- Preconditions.checkNotNull(transportProtos);
- PipelineChannel pipelineChannel =
- new PipelineChannel(transportProtos.getLeaderID(),
- transportProtos.getState(),
- transportProtos.getType(),
- transportProtos.getFactor(),
- transportProtos.getName());
-
- for (HddsProtos.DatanodeDetailsProto dataID :
- transportProtos.getMembersList()) {
- pipelineChannel.addMember(DatanodeDetails.getFromProtoBuf(dataID));
- }
- return pipelineChannel;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-hdds/common/src/main/proto/hdds.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
index a9a703e..816efa7 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -40,7 +40,7 @@ message Port {
required uint32 value = 2;
}
-message PipelineChannel {
+message Pipeline {
required string leaderID = 1;
repeated DatanodeDetailsProto members = 2;
optional LifeCycleState state = 3 [default = OPEN];
@@ -49,12 +49,6 @@ message PipelineChannel {
optional string name = 6;
}
-// A pipeline is composed of PipelineChannel (Ratis/StandAlone) that back a
-// container.
-message Pipeline {
- required PipelineChannel pipelineChannel = 2;
-}
-
message KeyValue {
required string key = 1;
optional string value = 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java
index 937076c..cbb2ba7 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java
@@ -127,12 +127,12 @@ public class ContainerCloser {
// to SCM. In that case also, data node will ignore this command.
HddsProtos.Pipeline pipeline = info.getPipeline();
- for (HddsProtos.DatanodeDetailsProto datanodeDetails : pipeline
- .getPipelineChannel().getMembersList()) {
+ for (HddsProtos.DatanodeDetailsProto datanodeDetails :
+ pipeline.getMembersList()) {
nodeManager.addDatanodeCommand(
DatanodeDetails.getFromProtoBuf(datanodeDetails).getUuid(),
new CloseContainerCommand(info.getContainerID(),
- pipeline.getPipelineChannel().getType()));
+ pipeline.getType()));
}
if (!commandIssued.containsKey(info.getContainerID())) {
commandIssued.put(info.getContainerID(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java
index 832fcc6..48affa4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.hdds.scm.pipelines;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineChannel;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -36,12 +35,12 @@ import java.util.concurrent.atomic.AtomicInteger;
public abstract class PipelineManager {
private static final Logger LOG =
LoggerFactory.getLogger(PipelineManager.class);
- private final List<PipelineChannel> activePipelineChannels;
- private final AtomicInteger conduitsIndex;
+ private final List<Pipeline> activePipelines;
+ private final AtomicInteger pipelineIndex;
public PipelineManager() {
- activePipelineChannels = new LinkedList<>();
- conduitsIndex = new AtomicInteger(0);
+ activePipelines = new LinkedList<>();
+ pipelineIndex = new AtomicInteger(0);
}
/**
@@ -59,9 +58,9 @@ public abstract class PipelineManager {
/**
* In the Ozone world, we have a very simple policy.
*
- * 1. Try to create a pipelineChannel if there are enough free nodes.
+ * 1. Try to create a pipeline if there are enough free nodes.
*
- * 2. This allows all nodes to part of a pipelineChannel quickly.
+ * 2. This allows all nodes to part of a pipeline quickly.
*
* 3. if there are not enough free nodes, return conduits in a
* round-robin fashion.
@@ -70,28 +69,28 @@ public abstract class PipelineManager {
* Create a new placement policy that returns conduits in round robin
* fashion.
*/
- PipelineChannel pipelineChannel =
- allocatePipelineChannel(replicationFactor);
- if (pipelineChannel != null) {
- LOG.debug("created new pipelineChannel:{} for container with " +
+ Pipeline pipeline =
+ allocatePipeline(replicationFactor);
+ if (pipeline != null) {
+ LOG.debug("created new pipeline:{} for container with " +
"replicationType:{} replicationFactor:{}",
- pipelineChannel.getName(), replicationType, replicationFactor);
- activePipelineChannels.add(pipelineChannel);
+ pipeline.getPipelineName(), replicationType, replicationFactor);
+ activePipelines.add(pipeline);
} else {
- pipelineChannel =
- findOpenPipelineChannel(replicationType, replicationFactor);
- if (pipelineChannel != null) {
- LOG.debug("re-used pipelineChannel:{} for container with " +
+ pipeline =
+ findOpenPipeline(replicationType, replicationFactor);
+ if (pipeline != null) {
+ LOG.debug("re-used pipeline:{} for container with " +
"replicationType:{} replicationFactor:{}",
- pipelineChannel.getName(), replicationType, replicationFactor);
+ pipeline.getPipelineName(), replicationType, replicationFactor);
}
}
- if (pipelineChannel == null) {
- LOG.error("Get pipelineChannel call failed. We are not able to find" +
- "free nodes or operational pipelineChannel.");
+ if (pipeline == null) {
+ LOG.error("Get pipeline call failed. We are not able to find" +
+ "free nodes or operational pipeline.");
return null;
} else {
- return new Pipeline(pipelineChannel);
+ return pipeline;
}
}
@@ -106,19 +105,19 @@ public abstract class PipelineManager {
}
}
- public abstract PipelineChannel allocatePipelineChannel(
+ public abstract Pipeline allocatePipeline(
ReplicationFactor replicationFactor) throws IOException;
/**
- * Find a PipelineChannel that is operational.
+ * Find a Pipeline that is operational.
*
* @return - Pipeline or null
*/
- private PipelineChannel findOpenPipelineChannel(
+ private Pipeline findOpenPipeline(
ReplicationType type, ReplicationFactor factor) {
- PipelineChannel pipelineChannel = null;
+ Pipeline pipeline = null;
final int sentinal = -1;
- if (activePipelineChannels.size() == 0) {
+ if (activePipelines.size() == 0) {
LOG.error("No Operational conduits found. Returning null.");
return null;
}
@@ -126,26 +125,26 @@ public abstract class PipelineManager {
int nextIndex = sentinal;
for (; startIndex != nextIndex; nextIndex = getNextIndex()) {
// Just walk the list in a circular way.
- PipelineChannel temp =
- activePipelineChannels
+ Pipeline temp =
+ activePipelines
.get(nextIndex != sentinal ? nextIndex : startIndex);
- // if we find an operational pipelineChannel just return that.
+ // if we find an operational pipeline just return that.
if ((temp.getLifeCycleState() == LifeCycleState.OPEN) &&
(temp.getFactor() == factor) && (temp.getType() == type)) {
- pipelineChannel = temp;
+ pipeline = temp;
break;
}
}
- return pipelineChannel;
+ return pipeline;
}
/**
- * gets the next index of the PipelineChannel to get.
+ * gets the next index of the Pipeline to get.
*
* @return index in the link list to get.
*/
private int getNextIndex() {
- return conduitsIndex.incrementAndGet() % activePipelineChannels.size();
+ return pipelineIndex.incrementAndGet() % activePipelines.size();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
index 2e56043..508ca9b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
@@ -20,7 +20,6 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineChannel;
import org.apache.hadoop.hdds.scm.container.placement.algorithms
.ContainerPlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms
@@ -85,20 +84,20 @@ public class PipelineSelector {
* The first of the list will be the leader node.
* @return pipeline corresponding to nodes
*/
- public static PipelineChannel newPipelineFromNodes(
+ public static Pipeline newPipelineFromNodes(
List<DatanodeDetails> nodes, LifeCycleState state,
ReplicationType replicationType, ReplicationFactor replicationFactor,
String name) {
Preconditions.checkNotNull(nodes);
Preconditions.checkArgument(nodes.size() > 0);
String leaderId = nodes.get(0).getUuidString();
- PipelineChannel
- pipelineChannel = new PipelineChannel(leaderId, state, replicationType,
+ Pipeline
+ pipeline = new Pipeline(leaderId, state, replicationType,
replicationFactor, name);
for (DatanodeDetails node : nodes) {
- pipelineChannel.addMember(node);
+ pipeline.addMember(node);
}
- return pipelineChannel;
+ return pipeline;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java
index 70489b9..ace8758 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java
@@ -20,7 +20,6 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.XceiverClientRatis;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineChannel;
import org.apache.hadoop.hdds.scm.container.placement.algorithms
.ContainerPlacementPolicy;
import org.apache.hadoop.hdds.scm.node.NodeManager;
@@ -68,12 +67,12 @@ public class RatisManagerImpl extends PipelineManager {
}
/**
- * Allocates a new ratis PipelineChannel from the free nodes.
+ * Allocates a new ratis Pipeline from the free nodes.
*
* @param factor - One or Three
* @return PipelineChannel.
*/
- public PipelineChannel allocatePipelineChannel(ReplicationFactor factor) {
+ public Pipeline allocatePipeline(ReplicationFactor factor) {
List<DatanodeDetails> newNodesList = new LinkedList<>();
List<DatanodeDetails> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
int count = getReplicationCount(factor);
@@ -87,22 +86,20 @@ public class RatisManagerImpl extends PipelineManager {
// once a datanode has been added to a pipeline, exclude it from
// further allocations
ratisMembers.addAll(newNodesList);
- LOG.info("Allocating a new ratis pipelineChannel of size: {}", count);
+ LOG.info("Allocating a new ratis pipeline of size: {}", count);
// Start all channel names with "Ratis", easy to grep the logs.
String conduitName = PREFIX +
UUID.randomUUID().toString().substring(PREFIX.length());
- PipelineChannel pipelineChannel =
+ Pipeline pipeline=
PipelineSelector.newPipelineFromNodes(newNodesList,
LifeCycleState.OPEN, ReplicationType.RATIS, factor, conduitName);
- Pipeline pipeline =
- new Pipeline(pipelineChannel);
try (XceiverClientRatis client =
XceiverClientRatis.newXceiverClientRatis(pipeline, conf)) {
client.createPipeline(pipeline.getPipelineName(), newNodesList);
} catch (IOException e) {
return null;
}
- return pipelineChannel;
+ return pipeline;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java
index 8268329..e76027f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java
@@ -17,7 +17,7 @@
package org.apache.hadoop.hdds.scm.pipelines.standalone;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineChannel;
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.scm.container.placement.algorithms
.ContainerPlacementPolicy;
import org.apache.hadoop.hdds.scm.node.NodeManager;
@@ -67,12 +67,12 @@ public class StandaloneManagerImpl extends PipelineManager {
/**
- * Allocates a new standalone PipelineChannel from the free nodes.
+ * Allocates a new standalone Pipeline from the free nodes.
*
* @param factor - One
- * @return PipelineChannel.
+ * @return Pipeline.
*/
- public PipelineChannel allocatePipelineChannel(ReplicationFactor factor) {
+ public Pipeline allocatePipeline(ReplicationFactor factor) {
List<DatanodeDetails> newNodesList = new LinkedList<>();
List<DatanodeDetails> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
int count = getReplicationCount(factor);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
index adb212a..d06d568 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerMapping;
import org.apache.hadoop.hdds.scm.container.Mapping;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineChannel;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -357,11 +356,10 @@ public class TestDeletedBlockLog {
private void mockContainerInfo(Mapping mappingService, long containerID,
DatanodeDetails dd) throws IOException {
- PipelineChannel pipelineChannel =
- new PipelineChannel("fake", LifeCycleState.OPEN,
+ Pipeline pipeline =
+ new Pipeline("fake", LifeCycleState.OPEN,
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, "fake");
- pipelineChannel.addMember(dd);
- Pipeline pipeline = new Pipeline(pipelineChannel);
+ pipeline.addMember(dd);
ContainerInfo.Builder builder = new ContainerInfo.Builder();
builder.setPipeline(pipeline);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index 0254984..50cdd54 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.XceiverClient;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineChannel;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.test.TestGenericTestUtils;
@@ -92,13 +91,12 @@ public class TestMiniOzoneCluster {
for(HddsDatanodeService dn : datanodes) {
// Create a single member pipe line
DatanodeDetails datanodeDetails = dn.getDatanodeDetails();
- final PipelineChannel pipelineChannel =
- new PipelineChannel(datanodeDetails.getUuidString(),
+ final Pipeline pipeline =
+ new Pipeline(datanodeDetails.getUuidString(),
HddsProtos.LifeCycleState.OPEN,
HddsProtos.ReplicationType.STAND_ALONE,
HddsProtos.ReplicationFactor.ONE, "test");
- pipelineChannel.addMember(datanodeDetails);
- Pipeline pipeline = new Pipeline(pipelineChannel);
+ pipeline.addMember(datanodeDetails);
// Verify client is able to connect to the container
try (XceiverClient client = new XceiverClient(pipeline, conf)){
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
index 7046132..459da2e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineChannel;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
@@ -136,14 +135,14 @@ public final class ContainerTestHelper {
Preconditions.checkArgument(i.hasNext());
final DatanodeDetails leader = i.next();
String pipelineName = "TEST-" + UUID.randomUUID().toString().substring(3);
- final PipelineChannel pipelineChannel =
- new PipelineChannel(leader.getUuidString(), LifeCycleState.OPEN,
+ final Pipeline pipeline =
+ new Pipeline(leader.getUuidString(), LifeCycleState.OPEN,
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, pipelineName);
- pipelineChannel.addMember(leader);
+ pipeline.addMember(leader);
for(; i.hasNext();) {
- pipelineChannel.addMember(i.next());
+ pipeline.addMember(i.next());
}
- return new Pipeline(pipelineChannel);
+ return pipeline;
}
/**
@@ -207,8 +206,6 @@ public final class ContainerTestHelper {
ContainerProtos.WriteChunkRequestProto
.newBuilder();
- Pipeline newPipeline =
- new Pipeline(pipeline.getPipelineChannel());
writeRequest.setBlockID(blockID.getDatanodeBlockIDProtobuf());
byte[] data = getData(datalen);
@@ -223,7 +220,7 @@ public final class ContainerTestHelper {
request.setCmdType(ContainerProtos.Type.WriteChunk);
request.setWriteChunk(writeRequest);
request.setTraceID(UUID.randomUUID().toString());
- request.setDatanodeUuid(newPipeline.getLeader().getUuidString());
+ request.setDatanodeUuid(pipeline.getLeader().getUuidString());
return request.build();
}
@@ -241,8 +238,6 @@ public final class ContainerTestHelper {
throws Exception {
ContainerProtos.PutSmallFileRequestProto.Builder smallFileRequest =
ContainerProtos.PutSmallFileRequestProto.newBuilder();
- Pipeline newPipeline =
- new Pipeline(pipeline.getPipelineChannel());
byte[] data = getData(dataLen);
ChunkInfo info = getChunk(blockID.getLocalID(), 0, 0, dataLen);
setDataChecksum(info, data);
@@ -266,7 +261,7 @@ public final class ContainerTestHelper {
request.setCmdType(ContainerProtos.Type.PutSmallFile);
request.setPutSmallFile(smallFileRequest);
request.setTraceID(UUID.randomUUID().toString());
- request.setDatanodeUuid(newPipeline.getLeader().getUuidString());
+ request.setDatanodeUuid(pipeline.getLeader().getUuidString());
return request.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
index 1b1153b..375450c 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineChannel;
import org.apache.hadoop.hdds.scm.container.states.ContainerStateMap;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.util.Time;
@@ -150,14 +149,14 @@ public class BenchMarkContainerStateMap {
Preconditions.checkArgument(i.hasNext());
final DatanodeDetails leader = i.next();
String pipelineName = "TEST-" + UUID.randomUUID().toString().substring(5);
- final PipelineChannel pipelineChannel =
- new PipelineChannel(leader.getUuidString(), OPEN,
+ final Pipeline pipeline =
+ new Pipeline(leader.getUuidString(), OPEN,
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, pipelineName);
- pipelineChannel.addMember(leader);
+ pipeline.addMember(leader);
for (; i.hasNext();) {
- pipelineChannel.addMember(i.next());
+ pipeline.addMember(i.next());
}
- return new Pipeline(pipelineChannel);
+ return pipeline;
}
@Benchmark
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index 13b04c3..3d4426f 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.genesis;
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.ratis.shaded.com.google.protobuf.ByteString;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
@@ -32,7 +33,6 @@ import org.apache.hadoop.ozone.container.common.impl.Dispatcher;
import org.apache.hadoop.ozone.container.common.impl.KeyManagerImpl;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineChannel;
import org.apache.hadoop.util.Time;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Level;
@@ -78,7 +78,7 @@ public class BenchMarkDatanodeDispatcher {
private String baseDir;
private String datanodeUuid;
private Dispatcher dispatcher;
- private PipelineChannel pipelineChannel;
+ private Pipeline pipeline;
private ByteString data;
private Random random;
private AtomicInteger containerCount;
@@ -96,7 +96,7 @@ public class BenchMarkDatanodeDispatcher {
@Setup(Level.Trial)
public void initialize() throws IOException {
datanodeUuid = UUID.randomUUID().toString();
- pipelineChannel = new PipelineChannel("127.0.0.1",
+ pipeline = new Pipeline("127.0.0.1",
LifeCycleState.OPEN, ReplicationType.STAND_ALONE,
ReplicationFactor.ONE, "SA-" + UUID.randomUUID());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a4632d3/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
index d4ac994..2bd43fb 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
@@ -519,11 +519,11 @@ public class SQLCLI extends Configured implements Tool {
LOG.info("Insert to sql container db, for container {}", containerID);
String insertContainerInfo = String.format(
INSERT_CONTAINER_INFO, containerID,
- pipeline.getPipelineChannel().getLeaderID());
+ pipeline.getLeaderID());
executeSQL(conn, insertContainerInfo);
for (HddsProtos.DatanodeDetailsProto dd :
- pipeline.getPipelineChannel().getMembersList()) {
+ pipeline.getMembersList()) {
String uuid = dd.getUuid();
if (!uuidChecked.contains(uuid)) {
// we may also not use this checked set, but catch exception instead
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HADOOP-15416. Clear error message
in S3Guard diff if source not found. Contributed by Gabor Bota.
Posted by xk...@apache.org.
HADOOP-15416. Clear error message in S3Guard diff if source not found. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55fad6a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55fad6a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55fad6a3
Branch: refs/heads/HDFS-12943
Commit: 55fad6a3de3125d9e7e2e9a5f8fa5b1b22a1de60
Parents: 6432128
Author: Sean Mackrory <ma...@apache.org>
Authored: Fri Jun 22 11:36:36 2018 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Fri Jun 22 11:36:56 2018 -0600
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55fad6a3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index fbffba9..ac10e08 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -805,7 +805,9 @@ public abstract class S3GuardTool extends Configured implements Tool {
*/
private void compareDir(FileStatus msDir, FileStatus s3Dir,
PrintStream out) throws IOException {
- Preconditions.checkArgument(!(msDir == null && s3Dir == null));
+ Preconditions.checkArgument(!(msDir == null && s3Dir == null),
+ "The path does not exist in metadata store and on s3.");
+
if (msDir != null && s3Dir != null) {
Preconditions.checkArgument(msDir.getPath().equals(s3Dir.getPath()),
String.format("The path from metadata store and s3 are different:" +
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDFS-13621. Upgrade commons-lang
version to 3.7 in hadoop-hdfs-project. Contributed by Takanobu Asanuma.
Posted by xk...@apache.org.
HDFS-13621. Upgrade commons-lang version to 3.7 in hadoop-hdfs-project. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fba9d7cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fba9d7cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fba9d7cd
Branch: refs/heads/HDFS-12943
Commit: fba9d7cd746cd7b659d2fd9d2bfa23266be9009b
Parents: 2a4632d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Jun 18 10:17:07 2018 -0700
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Jun 18 10:17:07 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/XAttr.java | 4 ++--
.../java/org/apache/hadoop/hdfs/ExtendedBlockId.java | 4 ++--
.../hadoop/hdfs/client/impl/BlockReaderFactory.java | 2 +-
.../hdfs/protocol/AddErasureCodingPolicyResponse.java | 4 ++--
.../apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java | 4 ++--
.../org/apache/hadoop/hdfs/protocol/CachePoolInfo.java | 4 ++--
.../org/apache/hadoop/hdfs/protocol/EncryptionZone.java | 4 ++--
.../hadoop/hdfs/protocol/ErasureCodingPolicy.java | 4 ++--
.../hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java | 4 ++--
.../org/apache/hadoop/hdfs/protocol/ExtendedBlock.java | 2 +-
.../hadoop/hdfs/shortcircuit/DfsClientShmManager.java | 2 +-
.../hadoop/hdfs/shortcircuit/ShortCircuitCache.java | 2 +-
.../hadoop/hdfs/shortcircuit/ShortCircuitShm.java | 4 ++--
.../namenode/ha/TestRequestHedgingProxyProvider.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 5 -----
.../hdfs/server/federation/router/ConnectionPoolId.java | 2 +-
.../server/federation/router/RemoteLocationContext.java | 2 +-
.../store/driver/impl/StateStoreFileImpl.java | 2 +-
.../server/federation/store/records/MountTable.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 -----
.../org/apache/hadoop/hdfs/protocol/CacheDirective.java | 2 +-
.../hdfs/qjournal/server/GetJournalEditServlet.java | 4 ++--
.../org/apache/hadoop/hdfs/qjournal/server/Journal.java | 12 ++++++------
.../hadoop/hdfs/server/datanode/DirectoryScanner.java | 2 +-
.../server/datanode/fsdataset/impl/FsDatasetCache.java | 2 +-
.../hdfs/server/diskbalancer/command/Command.java | 4 ++--
.../hdfs/server/diskbalancer/command/PlanCommand.java | 4 ++--
.../hdfs/server/diskbalancer/command/ReportCommand.java | 4 ++--
.../hdfs/server/namenode/EncryptionZoneManager.java | 4 ++--
.../hdfs/server/namenode/FSDirErasureCodingOp.java | 2 +-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 6 +++---
.../hdfs/server/namenode/INodeAttributeProvider.java | 2 +-
.../hdfs/server/namenode/startupprogress/Step.java | 6 +++---
.../hdfs/server/namenode/top/metrics/TopMetrics.java | 2 +-
.../java/org/apache/hadoop/hdfs/tools/CacheAdmin.java | 2 +-
.../java/org/apache/hadoop/hdfs/util/EnumCounters.java | 2 +-
.../apache/hadoop/fs/TestEnhancedByteBufferAccess.java | 4 ++--
.../test/java/org/apache/hadoop/fs/TestGlobPaths.java | 2 +-
.../hadoop/fs/TestWebHdfsFileContextMainOperations.java | 2 +-
.../test/java/org/apache/hadoop/hdfs/DFSTestUtil.java | 4 ++--
.../test/java/org/apache/hadoop/hdfs/TestDFSShell.java | 2 +-
.../java/org/apache/hadoop/hdfs/TestDecommission.java | 2 +-
.../org/apache/hadoop/hdfs/TestFsShellPermission.java | 2 +-
.../org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java | 2 +-
.../apache/hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +-
.../datatransfer/sasl/SaslDataTransferTestCase.java | 2 +-
.../hadoop/hdfs/server/balancer/TestBalancer.java | 2 +-
.../hadoop/hdfs/server/datanode/SimulatedFSDataset.java | 2 +-
.../diskbalancer/command/TestDiskBalancerCommand.java | 2 +-
.../hadoop/hdfs/server/namenode/FSImageTestUtil.java | 2 +-
.../hadoop/hdfs/server/namenode/NameNodeAdapter.java | 2 +-
.../hadoop/hdfs/server/namenode/TestAuditLogger.java | 2 +-
.../hdfs/server/namenode/TestCacheDirectives.java | 2 +-
.../server/namenode/TestEditLogJournalFailures.java | 2 +-
.../hadoop/hdfs/shortcircuit/TestShortCircuitCache.java | 2 +-
.../java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +-
.../java/org/apache/hadoop/tracing/TestTracing.java | 2 +-
57 files changed, 80 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/XAttr.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/XAttr.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/XAttr.java
index ad7b056..de9bbda 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/XAttr.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/XAttr.java
@@ -19,8 +19,8 @@ package org.apache.hadoop.fs;
import java.util.Arrays;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ExtendedBlockId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ExtendedBlockId.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ExtendedBlockId.java
index fe39df6..5dfcc73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ExtendedBlockId.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ExtendedBlockId.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.hdfs;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
index e83c8ae..a8c73a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
@@ -31,7 +31,7 @@ import java.util.List;
import com.google.common.io.ByteArrayDataOutput;
import com.google.common.io.ByteStreams;
-import org.apache.commons.lang.mutable.MutableBoolean;
+import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddErasureCodingPolicyResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddErasureCodingPolicyResponse.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddErasureCodingPolicyResponse.java
index dc77a47..f873b84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddErasureCodingPolicyResponse.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddErasureCodingPolicyResponse.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.hdfs.protocol;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.HadoopIllegalArgumentException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java
index d8a7de2..e80f12a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java
@@ -19,8 +19,8 @@ package org.apache.hadoop.hdfs.protocol;
import java.util.Date;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java
index daa77be..6c9f277 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java
@@ -22,8 +22,8 @@ import java.io.IOException;
import javax.annotation.Nullable;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.InvalidRequestException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java
index f1441b5..0b851ca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.hdfs.protocol;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.crypto.CipherSuite;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
index 39489b4..3559ab9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.hdfs.protocol;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java
index a5b95cb..c8a2722 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.hdfs.protocol;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java
index 7939662..8413c84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdfs.protocol;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
index 6f8a8fa..2262003 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
@@ -29,7 +29,7 @@ import java.util.TreeMap;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.commons.lang.mutable.MutableBoolean;
+import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.ExtendedBlockId;
import org.apache.hadoop.hdfs.net.DomainPeer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
index c2f0350..9c2d2e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
@@ -34,7 +34,7 @@ import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.collections.map.LinkedMap;
-import org.apache.commons.lang.mutable.MutableBoolean;
+import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.ExtendedBlockId;
import org.apache.hadoop.hdfs.client.impl.DfsClientConf.ShortCircuitConf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
index fb0e06f..b9fcada 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
@@ -25,8 +25,8 @@ import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Random;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.fs.InvalidRequestException;
import org.apache.hadoop.hdfs.ExtendedBlockId;
import org.apache.hadoop.io.nativeio.NativeIO;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
index 8913f1a..5c33ef6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
@@ -28,7 +28,7 @@ import java.util.Iterator;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index e9525e2..42d2c00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -124,11 +124,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<scope>compile</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
index 458fec2..868476a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java
@@ -24,7 +24,7 @@ import java.util.Collections;
import java.util.List;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.UserGroupInformation;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
index 0959eaa..cf78be3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteLocationContext.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
/**
* Base class for objects that are unique to a namespace.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
index 6b288b3..60dbcdc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
@@ -30,7 +30,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
index 005882e..49cdf10 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
@@ -26,7 +26,7 @@ import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.TreeMap;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index eaf9361..fcd5ae1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -114,11 +114,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<scope>compile</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java
index 89cf641..f8987a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java
@@ -21,7 +21,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
import java.util.Date;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSUtil;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
index e96fd4d..64ac11c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
@@ -31,7 +31,7 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -157,7 +157,7 @@ public class GetJournalEditServlet extends HttpServlet {
int myNsId = storage.getNamespaceID();
String myClusterId = storage.getClusterID();
- String theirStorageInfoString = StringEscapeUtils.escapeHtml(
+ String theirStorageInfoString = StringEscapeUtils.escapeHtml4(
request.getParameter(STORAGEINFO_PARAM));
if (theirStorageInfoString != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
index 452664a..8f25d26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
@@ -31,7 +31,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang.math.LongRange;
+import org.apache.commons.lang3.Range;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -842,8 +842,8 @@ public class Journal implements Closeable {
// Paranoid sanity check: if the new log is shorter than the log we
// currently have, we should not end up discarding any transactions
// which are already Committed.
- if (txnRange(currentSegment).containsLong(committedTxnId.get()) &&
- !txnRange(segment).containsLong(committedTxnId.get())) {
+ if (txnRange(currentSegment).contains(committedTxnId.get()) &&
+ !txnRange(segment).contains(committedTxnId.get())) {
throw new AssertionError(
"Cannot replace segment " +
TextFormat.shortDebugString(currentSegment) +
@@ -862,7 +862,7 @@ public class Journal implements Closeable {
// If we're shortening the log, update our highest txid
// used for lag metrics.
- if (txnRange(currentSegment).containsLong(highestWrittenTxId)) {
+ if (txnRange(currentSegment).contains(highestWrittenTxId)) {
updateHighestWrittenTxId(segment.getEndTxId());
}
}
@@ -906,10 +906,10 @@ public class Journal implements Closeable {
TextFormat.shortDebugString(newData) + " ; journal id: " + journalId);
}
- private LongRange txnRange(SegmentStateProto seg) {
+ private Range<Long> txnRange(SegmentStateProto seg) {
Preconditions.checkArgument(seg.hasEndTxId(),
"invalid segment: %s ; journal id: %s", seg, journalId);
- return new LongRange(seg.getStartTxId(), seg.getEndTxId());
+ return Range.between(seg.getStartTxId(), seg.getEndTxId());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 89f7c5d..39665e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -38,7 +38,7 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.commons.lang.time.FastDateFormat;
+import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
index f70d4af..767b150 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
@@ -43,7 +43,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.time.DurationFormatUtils;
+import org.apache.commons.lang3.time.DurationFormatUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.ChecksumException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
index 8eacdec..968a5a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
@@ -26,8 +26,8 @@ import com.google.common.collect.Lists;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.text.StrBuilder;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.text.StrBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.CommonConfigurationKeys;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
index b765885..90cc0c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
@@ -22,8 +22,8 @@ import com.google.common.base.Throwables;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.text.StrBuilder;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.text.StrBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
index 58ef5ce..5f4e0f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
@@ -24,8 +24,8 @@ import java.util.ListIterator;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.text.StrBuilder;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.text.StrBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerException;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerDataNode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index d06cd1c..5604a21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -34,8 +34,8 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 7160b86..769c137 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.XAttr;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index a8c1926..f94f6d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import static org.apache.commons.lang.StringEscapeUtils.escapeJava;
+import static org.apache.commons.lang3.StringEscapeUtils.escapeJava;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT;
@@ -1827,7 +1827,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
INodeFile inodeFile = ucFile.asFile();
String fullPathName = inodeFile.getFullPathName();
- if (org.apache.commons.lang.StringUtils.isEmpty(path)
+ if (org.apache.commons.lang3.StringUtils.isEmpty(path)
|| fullPathName.startsWith(path)) {
openFileEntries.add(new OpenFileEntry(inodeFile.getId(),
inodeFile.getFullPathName(),
@@ -2383,7 +2383,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean shouldReplicate = flag.contains(CreateFlag.SHOULD_REPLICATE);
if (shouldReplicate &&
- (!org.apache.commons.lang.StringUtils.isEmpty(ecPolicyName))) {
+ (!org.apache.commons.lang3.StringUtils.isEmpty(ecPolicyName))) {
throw new HadoopIllegalArgumentException("SHOULD_REPLICATE flag and " +
"ecPolicyName are exclusive parameters. Set both is not allowed!");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java
index 2f9bc37..8392463 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/Step.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/Step.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/Step.java
index 9b23e09..0baf99d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/Step.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/Step.java
@@ -18,9 +18,9 @@ package org.apache.hadoop.hdfs.server.namenode.startupprogress;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.lang.builder.CompareToBuilder;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.CompareToBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
index 2719c88..4d61d0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode.top.metrics;
import com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
index d8cbfc6..9781ea1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
@@ -22,7 +22,7 @@ import java.util.EnumSet;
import java.util.LinkedList;
import java.util.List;
-import org.apache.commons.lang.WordUtils;
+import org.apache.commons.lang3.text.WordUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java
index bec44a9..280a2d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hdfs.util;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import java.util.Arrays;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
index 9cd46c1..417d31b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
@@ -35,8 +35,8 @@ import java.util.Random;
import java.util.concurrent.TimeoutException;
import org.apache.commons.collections.map.LinkedMap;
-import org.apache.commons.lang.SystemUtils;
-import org.apache.commons.lang.mutable.MutableBoolean;
+import org.apache.commons.lang3.SystemUtils;
+import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.client.impl.BlockReaderTestUtil;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
index 32d960a..7027f3b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
@@ -26,7 +26,7 @@ import java.util.UUID;
import java.util.regex.Pattern;
import com.google.common.collect.Ordering;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.HdfsConfiguration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestWebHdfsFileContextMainOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestWebHdfsFileContextMainOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestWebHdfsFileContextMainOperations.java
index 72fc6e6..7544835 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestWebHdfsFileContextMainOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestWebHdfsFileContextMainOperations.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 63199f3..e6a2a00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -81,7 +81,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.UnhandledException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -2278,7 +2277,8 @@ public class DFSTestUtil {
", current value = " + currentValue);
return currentValue == expectedValue;
} catch (Exception e) {
- throw new UnhandledException("Test failed due to unexpected exception", e);
+ throw new RuntimeException(
+ "Test failed due to unexpected exception", e);
}
}
}, 1000, 60000);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index c352dc9..b19bdea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -37,7 +37,7 @@ import java.util.zip.GZIPOutputStream;
import com.google.common.base.Supplier;
import com.google.common.collect.Lists;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
index c0a595b..42b4257 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
@@ -38,7 +38,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.base.Supplier;
import com.google.common.collect.Lists;
-import org.apache.commons.lang.text.StrBuilder;
+import org.apache.commons.lang3.text.StrBuilder;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataOutputStream;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFsShellPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFsShellPermission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFsShellPermission.java
index cc456b2..7aa9f23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFsShellPermission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFsShellPermission.java
@@ -28,7 +28,7 @@ import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java
index e242690..3463f57 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java
@@ -28,7 +28,7 @@ import java.util.List;
import java.util.Set;
import com.google.common.collect.Sets;
-import org.apache.commons.lang.ClassUtils;
+import org.apache.commons.lang3.ClassUtils;
import org.apache.hadoop.hdfs.qjournal.server.JournalNodeRpcServer;
import org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
index 0b6bc6a..c87a6d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
-import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java
index d03d095..63ce45b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java
@@ -35,7 +35,7 @@ import static org.junit.Assert.*;
import java.io.File;
import java.util.Properties;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.http.HttpConfig;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index fa026f0..35ebe78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -73,7 +73,7 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeoutException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 987ba97..eb9461f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -38,7 +38,7 @@ import javax.management.ObjectName;
import javax.management.StandardMBean;
import com.google.common.math.LongMath;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.DF;
import org.apache.hadoop.fs.StorageType;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
index dee2a90..f2e998e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
@@ -43,7 +43,7 @@ import java.net.URI;
import java.util.List;
import java.util.Scanner;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
index 7be645f..f990c5e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
@@ -42,7 +42,7 @@ import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
index 3b601d5..b85527a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
@@ -24,7 +24,7 @@ import java.io.File;
import java.io.IOException;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.commons.lang.reflect.FieldUtils;
+import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
index 5b4f1f4..76cc906 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
index c58e090..551670e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
@@ -41,7 +41,7 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
-import org.apache.commons.lang.time.DateUtils;
+import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
index 28169bb..1e8ee9c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
@@ -29,7 +29,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
index 5da6a25..4e2cede 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
@@ -36,7 +36,7 @@ import java.util.concurrent.TimeoutException;
import net.jcip.annotations.NotThreadSafe;
import org.apache.commons.collections.map.LinkedMap;
-import org.apache.commons.lang.mutable.MutableBoolean;
+import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
index 647327c..1245247 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
@@ -27,7 +27,7 @@ import com.google.common.base.Supplier;
import com.google.common.collect.Lists;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.text.StrBuilder;
+import org.apache.commons.lang3.text.StrBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba9d7cd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
index bdad46a..04c85a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
@@ -21,7 +21,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataInputStream;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: YARN-7668. Remove unused variables
from ContainerLocalizer
Posted by xk...@apache.org.
YARN-7668. Remove unused variables from ContainerLocalizer
This closes #364
Signed-off-by: Akira Ajisaka <aa...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f386e78a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f386e78a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f386e78a
Branch: refs/heads/HDFS-12943
Commit: f386e78a4bc74e7c247b179c7d4ec27310fda4d3
Parents: 1e94e59
Author: Dedunu Dhananjaya <de...@gmail.com>
Authored: Mon Apr 23 09:34:57 2018 +0000
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Jun 18 16:45:50 2018 -0700
----------------------------------------------------------------------
.../nodemanager/containermanager/localizer/ContainerLocalizer.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f386e78a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index 639a69d..6a384ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -93,9 +93,7 @@ public class ContainerLocalizer {
public static final String FILECACHE = "filecache";
public static final String APPCACHE = "appcache";
public static final String USERCACHE = "usercache";
- public static final String OUTPUTDIR = "output";
public static final String TOKEN_FILE_NAME_FMT = "%s.tokens";
- public static final String WORKDIR = "work";
private static final String APPCACHE_CTXT_FMT = "%s.app.cache.dirs";
private static final String USERCACHE_CTXT_FMT = "%s.user.cache.dirs";
private static final FsPermission FILECACHE_PERMS =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: Merge branch 'trunk' into
HDFS-12943
Posted by xk...@apache.org.
Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/da14307c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/da14307c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/da14307c
Branch: refs/heads/HDFS-12943
Commit: da14307c354f5901c60e8eed97c1a793a62a8371
Parents: 292ccdc 238fe00
Author: Erik Krogen <xk...@apache.org>
Authored: Tue Jun 26 11:00:27 2018 +0530
Committer: Erik Krogen <xk...@apache.org>
Committed: Tue Jun 26 11:00:27 2018 +0530
----------------------------------------------------------------------
dev-support/bin/ozone-dist-layout-stitching | 4 +-
.../org/apache/hadoop/conf/Configuration.java | 8 +-
.../crypto/key/kms/KMSClientProvider.java | 16 +-
.../java/org/apache/hadoop/fs/BBPartHandle.java | 58 ++
.../org/apache/hadoop/fs/BBUploadHandle.java | 57 ++
.../fs/CommonConfigurationKeysPublic.java | 4 +-
.../hadoop/fs/FSDataOutputStreamBuilder.java | 22 +
.../java/org/apache/hadoop/fs/FileContext.java | 66 ++
.../hadoop/fs/FileSystemMultipartUploader.java | 132 ++++
.../hadoop/fs/LocalFileSystemPathHandle.java | 100 +++
.../org/apache/hadoop/fs/MultipartUploader.java | 90 +++
.../hadoop/fs/MultipartUploaderFactory.java | 65 ++
.../main/java/org/apache/hadoop/fs/Options.java | 3 +
.../java/org/apache/hadoop/fs/PartHandle.java | 45 ++
.../apache/hadoop/fs/RawLocalFileSystem.java | 61 +-
.../UnsupportedMultipartUploaderException.java | 41 ++
.../java/org/apache/hadoop/fs/UploadHandle.java | 47 ++
.../hadoop/metrics2/impl/MetricsConfig.java | 15 +-
.../security/ShellBasedUnixGroupsMapping.java | 10 +-
.../hadoop/security/UserGroupInformation.java | 4 +-
.../web/DelegationTokenAuthenticator.java | 8 +-
.../apache/hadoop/util/HttpExceptionUtils.java | 12 +-
.../apache/hadoop/util/JsonSerialization.java | 24 +
.../main/java/org/apache/hadoop/util/Shell.java | 2 +-
.../org/apache/hadoop/util/SysInfoLinux.java | 19 +-
.../hadoop-common/src/main/proto/FSProtos.proto | 8 +
...rg.apache.hadoop.fs.MultipartUploaderFactory | 16 +
.../fs/AbstractSystemMultipartUploaderTest.java | 143 ++++
.../fs/FileContextMainOperationsBaseTest.java | 44 +-
.../apache/hadoop/fs/TestLocalFileSystem.java | 23 +-
.../TestLocalFileSystemMultipartUploader.java | 65 ++
.../AbstractContractPathHandleTest.java | 6 +
.../TestRawlocalContractPathHandle.java | 40 ++
.../TestShellBasedUnixGroupsMapping.java | 39 +-
.../apache/hadoop/util/TestSysInfoLinux.java | 60 ++
.../src/test/resources/contract/rawlocal.xml | 5 +
.../test/scripts/process_with_sigterm_trap.sh | 24 +
.../crypto/key/kms/server/KMSJSONWriter.java | 6 +-
hadoop-dist/pom.xml | 5 +
.../container/common/helpers/ContainerInfo.java | 32 +
.../scm/container/common/helpers/Pipeline.java | 142 ++--
.../common/helpers/PipelineChannel.java | 124 ----
hadoop-hdds/common/src/main/proto/hdds.proto | 8 +-
.../statemachine/DatanodeStateMachine.java | 3 +
.../ReplicateContainerCommandHandler.java | 67 ++
.../states/endpoint/HeartbeatEndpointTask.java | 12 +
.../protocol/commands/CommandForDatanode.java | 45 ++
.../commands/ReplicateContainerCommand.java | 94 +++
.../StorageContainerDatanodeProtocol.proto | 12 +-
.../scm/container/closer/ContainerCloser.java | 6 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 20 +-
.../hdds/scm/pipelines/PipelineManager.java | 67 +-
.../hdds/scm/pipelines/PipelineSelector.java | 11 +-
.../scm/pipelines/ratis/RatisManagerImpl.java | 13 +-
.../standalone/StandaloneManagerImpl.java | 8 +-
.../scm/server/SCMDatanodeProtocolServer.java | 11 +
.../scm/server/StorageContainerManager.java | 8 +-
.../hdds/scm/block/TestDeletedBlockLog.java | 8 +-
.../hadoop/hdds/scm/node/TestNodeManager.java | 39 ++
.../main/java/org/apache/hadoop/fs/XAttr.java | 4 +-
.../hdfs/DFSMultipartUploaderFactory.java | 40 ++
.../org/apache/hadoop/hdfs/ExtendedBlockId.java | 4 +-
.../hdfs/client/impl/BlockReaderFactory.java | 2 +-
.../AddErasureCodingPolicyResponse.java | 4 +-
.../hdfs/protocol/CacheDirectiveInfo.java | 4 +-
.../hadoop/hdfs/protocol/CachePoolInfo.java | 4 +-
.../hadoop/hdfs/protocol/EncryptionZone.java | 4 +-
.../hdfs/protocol/ErasureCodingPolicy.java | 4 +-
.../hdfs/protocol/ErasureCodingPolicyInfo.java | 4 +-
.../hadoop/hdfs/protocol/ExtendedBlock.java | 2 +-
.../hdfs/shortcircuit/DfsClientShmManager.java | 2 +-
.../hdfs/shortcircuit/ShortCircuitCache.java | 2 +-
.../hdfs/shortcircuit/ShortCircuitShm.java | 4 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 7 +-
...onfRefreshTokenBasedAccessTokenProvider.java | 8 +-
.../CredentialBasedAccessTokenProvider.java | 8 +-
...rg.apache.hadoop.fs.MultipartUploaderFactory | 16 +
.../ha/TestRequestHedgingProxyProvider.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 5 -
.../federation/router/ConnectionPoolId.java | 2 +-
.../router/RemoteLocationContext.java | 2 +-
.../store/driver/impl/StateStoreFileImpl.java | 2 +-
.../federation/store/records/MountTable.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 -
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +-
.../hadoop/hdfs/protocol/CacheDirective.java | 2 +-
.../qjournal/server/GetJournalEditServlet.java | 4 +-
.../hadoop/hdfs/qjournal/server/Journal.java | 12 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 6 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 30 +-
.../server/blockmanagement/BlockManager.java | 2 +-
.../hdfs/server/datanode/DirectoryScanner.java | 4 +-
.../datanode/fsdataset/impl/FsDatasetCache.java | 2 +-
.../server/diskbalancer/command/Command.java | 4 +-
.../diskbalancer/command/PlanCommand.java | 4 +-
.../diskbalancer/command/ReportCommand.java | 4 +-
.../server/namenode/EncryptionZoneManager.java | 4 +-
.../server/namenode/FSDirErasureCodingOp.java | 2 +-
.../hdfs/server/namenode/FSNamesystem.java | 6 +-
.../hdfs/server/namenode/FSNamesystemLock.java | 17 +-
.../server/namenode/INodeAttributeProvider.java | 2 +-
.../server/namenode/startupprogress/Step.java | 6 +-
.../server/namenode/top/metrics/TopMetrics.java | 2 +-
.../apache/hadoop/hdfs/tools/CacheAdmin.java | 2 +-
.../apache/hadoop/hdfs/util/EnumCounters.java | 2 +-
.../src/main/resources/hdfs-default.xml | 10 +
.../hadoop/fs/TestEnhancedByteBufferAccess.java | 4 +-
.../org/apache/hadoop/fs/TestGlobPaths.java | 2 +-
.../hadoop/fs/TestHDFSMultipartUploader.java | 76 ++
.../TestWebHdfsFileContextMainOperations.java | 2 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 +-
.../org/apache/hadoop/hdfs/TestDFSShell.java | 2 +-
.../apache/hadoop/hdfs/TestDecommission.java | 2 +-
.../hadoop/hdfs/TestFsShellPermission.java | 2 +-
.../hadoop/hdfs/TestHDFSPolicyProvider.java | 2 +-
.../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +-
.../hdfs/TestSecureEncryptionZoneWithKMS.java | 34 +-
.../sasl/SaslDataTransferTestCase.java | 2 +-
.../hdfs/server/balancer/TestBalancer.java | 81 ++-
.../server/datanode/SimulatedFSDataset.java | 2 +-
.../command/TestDiskBalancerCommand.java | 2 +-
.../hadoop/hdfs/server/mover/TestMover.java | 46 ++
.../hdfs/server/namenode/FSImageTestUtil.java | 2 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
.../hdfs/server/namenode/TestAuditLogger.java | 2 +-
.../server/namenode/TestCacheDirectives.java | 2 +-
.../namenode/TestEditLogJournalFailures.java | 2 +-
.../hdfs/server/namenode/TestEditLogRace.java | 6 +-
.../server/namenode/TestFSNamesystemLock.java | 10 +-
.../hdfs/server/namenode/TestStartup.java | 9 +-
.../namenode/metrics/TestNameNodeMetrics.java | 4 +-
.../shortcircuit/TestShortCircuitCache.java | 2 +-
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +-
.../org/apache/hadoop/tracing/TestTracing.java | 2 +-
.../v2/app/job/impl/TestTaskAttempt.java | 4 +-
.../apache/hadoop/mapreduce/JobSubmitter.java | 8 +-
.../lib/input/CombineFileInputFormat.java | 4 +-
.../mapred/uploader/FrameworkUploader.java | 2 +-
.../mapred/uploader/TestFrameworkUploader.java | 13 +
hadoop-maven-plugins/pom.xml | 38 +-
.../maven/plugin/resourcegz/ResourceGzMojo.java | 2 +-
.../test/acceptance/ozonefs/docker-compose.yaml | 71 ++
.../src/test/acceptance/ozonefs/docker-config | 39 ++
.../src/test/acceptance/ozonefs/ozonefs.robot | 39 ++
.../hadoop/ozone/TestMiniOzoneCluster.java | 8 +-
.../ozone/container/ContainerTestHelper.java | 19 +-
.../TestReplicateContainerHandler.java | 71 ++
...TestGenerateOzoneRequiredConfigurations.java | 2 +-
.../ozone-manager/src/main/webapps/ksm/ksm.js | 4 +-
hadoop-ozone/ozonefs/pom.xml | 211 ++++++
.../org/apache/hadoop/fs/ozone/Constants.java | 42 ++
.../java/org/apache/hadoop/fs/ozone/OzFs.java | 44 ++
.../hadoop/fs/ozone/OzoneFSInputStream.java | 79 +++
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 59 ++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 +++++++++++++++++++
.../apache/hadoop/fs/ozone/package-info.java | 30 +
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 +++++
.../fs/ozone/TestOzoneFileInterfaces.java | 231 +++++++
.../contract/ITestOzoneContractCreate.java | 48 ++
.../contract/ITestOzoneContractDelete.java | 48 ++
.../contract/ITestOzoneContractDistCp.java | 50 ++
.../ITestOzoneContractGetFileStatus.java | 61 ++
.../ozone/contract/ITestOzoneContractMkdir.java | 48 ++
.../ozone/contract/ITestOzoneContractOpen.java | 47 ++
.../contract/ITestOzoneContractRename.java | 49 ++
.../contract/ITestOzoneContractRootDir.java | 51 ++
.../ozone/contract/ITestOzoneContractSeek.java | 47 ++
.../hadoop/fs/ozone/contract/OzoneContract.java | 123 ++++
.../src/test/resources/contract/ozone.xml | 113 +++
.../ozonefs/src/test/resources/log4j.properties | 23 +
hadoop-ozone/pom.xml | 1 +
.../genesis/BenchMarkContainerStateMap.java | 11 +-
.../genesis/BenchMarkDatanodeDispatcher.java | 6 +-
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 4 +-
hadoop-project/pom.xml | 11 +-
hadoop-tools/hadoop-aws/pom.xml | 42 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 11 +
.../hadoop/fs/s3a/S3AMultipartUploader.java | 150 ++++
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 1 +
.../fs/s3a/s3guard/LocalMetadataEntry.java | 81 +++
.../fs/s3a/s3guard/LocalMetadataStore.java | 247 ++++---
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 4 +-
...rg.apache.hadoop.fs.MultipartUploaderFactory | 15 +
.../org.apache.hadoop.fs.MultipartUploader | 16 +
.../hadoop/fs/s3a/AbstractS3ATestBase.java | 21 +-
.../apache/hadoop/fs/s3a/S3ATestConstants.java | 1 -
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 34 +-
.../fs/s3a/commit/staging/StagingTestBase.java | 30 +-
.../s3a/s3guard/DynamoDBLocalClientFactory.java | 160 -----
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 6 +-
.../s3a/s3guard/TestDynamoDBMetadataStore.java | 589 ----------------
.../fs/s3a/s3guard/TestLocalMetadataStore.java | 33 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 2 +-
.../hadoop/fs/azure/security/JsonUtils.java | 4 +-
hadoop-tools/hadoop-ozone/pom.xml | 174 -----
.../org/apache/hadoop/fs/ozone/Constants.java | 42 --
.../java/org/apache/hadoop/fs/ozone/OzFs.java | 44 --
.../hadoop/fs/ozone/OzoneFSInputStream.java | 79 ---
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 59 --
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 -------------------
.../apache/hadoop/fs/ozone/package-info.java | 30 -
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 -----
.../fs/ozone/TestOzoneFileInterfaces.java | 231 -------
.../contract/ITestOzoneContractCreate.java | 48 --
.../contract/ITestOzoneContractDelete.java | 48 --
.../contract/ITestOzoneContractDistCp.java | 50 --
.../ITestOzoneContractGetFileStatus.java | 61 --
.../ozone/contract/ITestOzoneContractMkdir.java | 48 --
.../ozone/contract/ITestOzoneContractOpen.java | 47 --
.../contract/ITestOzoneContractRename.java | 49 --
.../contract/ITestOzoneContractRootDir.java | 51 --
.../ozone/contract/ITestOzoneContractSeek.java | 47 --
.../hadoop/fs/ozone/contract/OzoneContract.java | 123 ----
.../src/test/resources/contract/ozone.xml | 113 ---
.../src/test/resources/log4j.properties | 23 -
hadoop-tools/hadoop-tools-dist/pom.xml | 15 -
hadoop-tools/pom.xml | 11 -
.../yarn/api/records/ResourceRequest.java | 20 +
.../hadoop/yarn/conf/YarnConfiguration.java | 2 +-
.../exceptions/RestApiErrorMessages.java | 2 +
.../yarn/service/utils/ServiceApiUtil.java | 5 +
.../hadoop/yarn/service/TestServiceApiUtil.java | 18 +
.../yarn/client/api/impl/AMRMClientImpl.java | 11 +-
.../api/impl/TestAMRMClientOnRMRestart.java | 6 +-
.../yarn/client/api/impl/TestYarnClient.java | 337 ++-------
.../client/api/impl/TestYarnClientImpl.java | 254 +++++++
.../hadoop/yarn/server/AMRMClientRelayer.java | 8 +-
.../LocalityMulticastAMRMProxyPolicy.java | 10 +-
.../server/scheduler/ResourceRequestSet.java | 14 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 12 -
.../src/CMakeLists.txt | 4 +-
.../server/nodemanager/ContainerExecutor.java | 4 +-
.../launcher/ContainerLaunch.java | 46 +-
.../launcher/ContainerRelaunch.java | 1 -
.../localizer/ContainerLocalizer.java | 7 +-
.../localizer/ResourceLocalizationService.java | 5 +-
.../oom-listener/test/oom_listener_test_main.cc | 33 +-
.../container/TestContainer.java | 8 +-
.../launcher/TestContainerLaunch.java | 2 -
.../scheduler/capacity/CSQueueUtils.java | 18 +-
.../fair/AllocationFileLoaderService.java | 4 +-
.../LocalityAppPlacementAllocator.java | 10 +-
.../webapp/dao/ClusterMetricsInfo.java | 2 +-
.../server/resourcemanager/Application.java | 10 +-
.../server/resourcemanager/TestAppManager.java | 18 +-
.../src/site/markdown/FairScheduler.md | 2 +-
.../hadoop-yarn-ui/src/main/webapp/.bowerrc | 3 +-
pom.xml | 2 +-
248 files changed, 5735 insertions(+), 4147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da14307c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da14307c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da14307c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da14307c/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HDFS-13682. Cannot create
encryption zone after KMS auth token expires.
Posted by xk...@apache.org.
HDFS-13682. Cannot create encryption zone after KMS auth token expires.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/32f867a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/32f867a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/32f867a6
Branch: refs/heads/HDFS-12943
Commit: 32f867a6a907c05a312657139d295a92756d98ef
Parents: b089a06
Author: Xiao Chen <xi...@apache.org>
Authored: Wed Jun 20 15:58:01 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Wed Jun 20 15:58:26 2018 -0700
----------------------------------------------------------------------
.../crypto/key/kms/KMSClientProvider.java | 9 ++++--
.../hadoop/security/UserGroupInformation.java | 4 ++-
.../hdfs/TestSecureEncryptionZoneWithKMS.java | 34 ++++++++++++++++++--
3 files changed, 41 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/32f867a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 08787a5..edbf897 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -32,7 +32,9 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -543,7 +545,9 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
String requestMethod = conn.getRequestMethod();
URL url = conn.getURL();
conn = createConnection(url, requestMethod);
- conn.setRequestProperty(CONTENT_TYPE, contentType);
+ if (contentType != null && !contentType.isEmpty()) {
+ conn.setRequestProperty(CONTENT_TYPE, contentType);
+ }
return call(conn, jsonOutput, expectedResponse, klass,
authRetryCount - 1);
}
@@ -1087,8 +1091,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
actualUgi = currentUgi.getRealUser();
}
if (UserGroupInformation.isSecurityEnabled() &&
- !containsKmsDt(actualUgi) &&
- !actualUgi.hasKerberosCredentials()) {
+ !containsKmsDt(actualUgi) && !actualUgi.shouldRelogin()) {
// Use login user is only necessary when Kerberos is enabled
// but the actual user does not have either
// Kerberos credential or KMS delegation token for KMS operations
http://git-wip-us.apache.org/repos/asf/hadoop/blob/32f867a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 3872810..29b9fea 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -831,7 +831,9 @@ public class UserGroupInformation {
return start + (long) ((end - start) * TICKET_RENEW_WINDOW);
}
- private boolean shouldRelogin() {
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public boolean shouldRelogin() {
return hasKerberosCredentials() && isHadoopLogin();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/32f867a6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java
index 7c4763c..db97c02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java
@@ -107,6 +107,8 @@ public class TestSecureEncryptionZoneWithKMS {
// MiniKMS
private static MiniKMS miniKMS;
private final String testKey = "test_key";
+ private static boolean testKeyCreated = false;
+ private static final long AUTH_TOKEN_VALIDITY = 1;
// MiniDFS
private MiniDFSCluster cluster;
@@ -128,7 +130,7 @@ public class TestSecureEncryptionZoneWithKMS {
}
@Rule
- public Timeout timeout = new Timeout(30000);
+ public Timeout timeout = new Timeout(120000);
@BeforeClass
public static void init() throws Exception {
@@ -215,6 +217,9 @@ public class TestSecureEncryptionZoneWithKMS {
"HTTP/localhost");
kmsConf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
kmsConf.set("hadoop.kms.acl.GENERATE_EEK", "hdfs");
+ // set kms auth token expiration low for testCreateZoneAfterAuthTokenExpiry
+ kmsConf.setLong("hadoop.kms.authentication.token.validity",
+ AUTH_TOKEN_VALIDITY);
Writer writer = new FileWriter(kmsFile);
kmsConf.writeXml(writer);
@@ -260,7 +265,10 @@ public class TestSecureEncryptionZoneWithKMS {
cluster.waitActive();
// Create a test key
- DFSTestUtil.createKey(testKey, cluster, conf);
+ if (!testKeyCreated) {
+ DFSTestUtil.createKey(testKey, cluster, conf);
+ testKeyCreated = true;
+ }
}
@After
@@ -307,4 +315,26 @@ public class TestSecureEncryptionZoneWithKMS {
}
});
}
+
+ @Test
+ public void testCreateZoneAfterAuthTokenExpiry() throws Exception {
+ final UserGroupInformation ugi = UserGroupInformation
+ .loginUserFromKeytabAndReturnUGI(hdfsPrincipal, keytab);
+ LOG.info("Created ugi: {} ", ugi);
+
+ ugi.doAs((PrivilegedExceptionAction<Object>) () -> {
+ final Path zone = new Path("/expire1");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, testKey, NO_TRASH);
+
+ final Path zone1 = new Path("/expire2");
+ fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true);
+ final long sleepInterval = (AUTH_TOKEN_VALIDITY + 1) * 1000;
+ LOG.info("Sleeping {} seconds to wait for kms auth token expiration",
+ sleepInterval);
+ Thread.sleep(sleepInterval);
+ dfsAdmin.createEncryptionZone(zone1, testKey, NO_TRASH);
+ return null;
+ });
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: YARN-8437. Build oom-listener
fails on older versions. (Miklos Szegedi via Haibo Chen)
Posted by xk...@apache.org.
YARN-8437. Build oom-listener fails on older versions. (Miklos Szegedi via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4939ffed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4939ffed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4939ffed
Branch: refs/heads/HDFS-12943
Commit: 4939ffedb151ce1550fcdd7ac04c79d8d0195891
Parents: 2d87592
Author: Haibo Chen <ha...@apache.org>
Authored: Wed Jun 20 10:35:52 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Wed Jun 20 10:42:13 2018 -0700
----------------------------------------------------------------------
.../src/CMakeLists.txt | 4 +--
.../oom-listener/test/oom_listener_test_main.cc | 33 ++++++++++----------
2 files changed, 19 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4939ffed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index a614f80..300bb65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -188,5 +188,5 @@ add_executable(test-oom-listener
main/native/oom-listener/impl/oom_listener.h
main/native/oom-listener/test/oom_listener_test_main.cc
)
-target_link_libraries(test-oom-listener gtest)
-output_directory(test-oom-listener test)
\ No newline at end of file
+target_link_libraries(test-oom-listener gtest rt)
+output_directory(test-oom-listener test)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4939ffed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/oom-listener/test/oom_listener_test_main.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/oom-listener/test/oom_listener_test_main.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/oom-listener/test/oom_listener_test_main.cc
index 9627632..421c21e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/oom-listener/test/oom_listener_test_main.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/oom-listener/test/oom_listener_test_main.cc
@@ -20,6 +20,7 @@
extern "C" {
#include "oom_listener.h"
+#include <time.h>
}
#include <gtest/gtest.h>
@@ -49,10 +50,10 @@ int main(int argc, char **argv) {
class OOMListenerTest : public ::testing::Test {
private:
- char cgroup[PATH_MAX] = {};
- const char* cgroup_root = nullptr;
+ char cgroup[PATH_MAX];
+ const char* cgroup_root;
public:
- OOMListenerTest() = default;
+ OOMListenerTest() : cgroup_root(NULL) {}
virtual ~OOMListenerTest() = default;
virtual const char* GetCGroup() { return cgroup; }
@@ -99,7 +100,7 @@ public:
if (cgroup[0] != '\0') {
rmdir(cgroup);
}
- if (cgroup_root != nullptr &&
+ if (cgroup_root != NULL &&
cgroup_root != cgroup_candidates[0]) {
rmdir(cgroup_root);
}
@@ -184,7 +185,7 @@ TEST_F(OOMListenerTest, test_oom) {
std::cout << "Consuming too much memory" << std::endl;
for (;;) {
auto buffer = (char *) malloc(bufferSize);
- if (buffer != nullptr) {
+ if (buffer != NULL) {
for (int i = 0; i < bufferSize; ++i) {
buffer[i] = (char) std::rand();
}
@@ -213,15 +214,15 @@ TEST_F(OOMListenerTest, test_oom) {
if (listener == 0) {
// child listener forwarding cgroup events
_oom_listener_descriptors descriptors = {
- .command = "test",
- .event_fd = mock_oom_event_as_user,
- .event_control_fd = -1,
- .oom_control_fd = -1,
- .event_control_path = {0},
- .oom_control_path = {0},
- .oom_command = {0},
- .oom_command_len = 0,
- .watch_timeout = 100
+ "test",
+ mock_oom_event_as_user,
+ -1,
+ -1,
+ {0},
+ {0},
+ {0},
+ 0,
+ 100
};
int ret = oom_listener(&descriptors, GetCGroup(), test_pipe[1]);
cleanup(&descriptors);
@@ -256,7 +257,7 @@ TEST_F(OOMListenerTest, test_oom) {
__pid_t exited0 = wait(mem_hog_status);
ASSERT_EQ(mem_hog_pid, exited0)
<< "Wrong process exited";
- ASSERT_EQ(nullptr, mem_hog_status)
+ ASSERT_EQ(NULL, mem_hog_status)
<< "Test process killed with invalid status";
if (mock_oom_event_as_user != -1) {
@@ -275,7 +276,7 @@ TEST_F(OOMListenerTest, test_oom) {
__pid_t exited1 = wait(oom_listener_status);
ASSERT_EQ(listener, exited1)
<< "Wrong process exited";
- ASSERT_EQ(nullptr, oom_listener_status)
+ ASSERT_EQ(NULL, oom_listener_status)
<< "Listener process exited with invalid status";
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HADOOP-15533. Make WASB listStatus
messages consistent. Contributed by Esfandiar Manii
Posted by xk...@apache.org.
HADOOP-15533. Make WASB listStatus messages consistent. Contributed by Esfandiar Manii
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3474460
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3474460
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3474460
Branch: refs/heads/HDFS-12943
Commit: f34744603ee93e082e7ba148df1400af5ac7c30c
Parents: 980031b
Author: Chris Douglas <cd...@apache.org>
Authored: Sun Jun 17 23:12:18 2018 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Sun Jun 17 23:12:18 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3474460/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index e05327e..dfc881a 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -2886,7 +2886,7 @@ public class NativeAzureFileSystem extends FileSystem {
// There is no metadata found for the path.
LOG.debug("Did not find any metadata for path: {}", key);
- throw new FileNotFoundException("File" + f + " does not exist.");
+ throw new FileNotFoundException(f + " is not found");
}
return status.toArray(new FileStatus[0]);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-8442. Strange characters and
missing spaces in FairScheduler documentation. Contributed by Szilard Nemeth.
Posted by xk...@apache.org.
YARN-8442. Strange characters and missing spaces in FairScheduler documentation. Contributed by Szilard Nemeth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/388fafa0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/388fafa0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/388fafa0
Branch: refs/heads/HDFS-12943
Commit: 388fafa004dc405a4e10f4487cff7c5a714af32f
Parents: bbbc7cc
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Wed Jun 20 11:55:43 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Wed Jun 20 11:55:43 2018 -0700
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/FairScheduler.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/388fafa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index e253d0d..269f5b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -148,7 +148,7 @@ The allocation file must be in XML format. The format contains five types of ele
* **secondaryGroupExistingQueue**: the app is placed into a queue with a name that matches a secondary group of the user who submitted it. The first secondary group that matches a configured queue will be selected. Periods in group names will be replaced with "\_dot\_", i.e. a user with "one.two" as one of their secondary groups would be placed into the "one\_dot\_two" queue, if such a queue exists.
- * **nestedUserQueue**: the app is placed into a queue with the name of the user under the queue suggested by the nested rule. This is similar to ‘user’ rule,the difference being in 'nestedUserQueue' rule,user queues can be created under any parent queue, while 'user' rule creates user queues only under root queue. Note that nestedUserQueue rule would be applied only if the nested rule returns a parent queue.One can configure a parent queue either by setting 'type' attribute of queue to 'parent' or by configuring at least one leaf under that queue which makes it a parent. See example allocation for a sample use case.
+ * **nestedUserQueue**: the app is placed into a queue with the name of the user under the queue suggested by the nested rule. This is similar to the 'user' rule, the difference being in 'nestedUserQueue' rule, user queues can be created under any parent queue, while 'user' rule creates user queues only under root queue. Note that nestedUserQueue rule would be applied only if the nested rule returns a parent queue. One can configure a parent queue either by setting 'type' attribute of queue to 'parent' or by configuring at least one leaf under that queue which makes it a parent. See example allocation for a sample use case.
* **default**: the app is placed into the queue specified in the 'queue' attribute of the default rule. If 'queue' attribute is not specified, the app is placed into 'root.default' queue.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: MAPREDUCE-7114. Make
FrameworkUploader symlink ignore improvement. Contributed by Gergo Repas.
Posted by xk...@apache.org.
MAPREDUCE-7114. Make FrameworkUploader symlink ignore improvement. Contributed by Gergo Repas.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae055622
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae055622
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae055622
Branch: refs/heads/HDFS-12943
Commit: ae055622edeb3cbf82baa6ed952fc2abc84c021e
Parents: 55fad6a
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Fri Jun 22 13:05:41 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Fri Jun 22 13:05:41 2018 -0700
----------------------------------------------------------------------
.../hadoop/mapred/uploader/FrameworkUploader.java | 2 +-
.../hadoop/mapred/uploader/TestFrameworkUploader.java | 13 +++++++++++++
2 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae055622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
index 5316f38..d2116c0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
@@ -409,7 +409,7 @@ public class FrameworkUploader implements Runnable {
linkPath == null ? null : linkPath.getParent();
java.nio.file.Path normalizedLinkPath =
linkPathParent == null ? null : linkPathParent.normalize();
- if (normalizedLinkPath != null && jarParent.equals(
+ if (normalizedLinkPath != null && jarParent.normalize().equals(
normalizedLinkPath)) {
LOG.info(String.format("Ignoring same directory link %s to %s",
jarPath.toString(), link.toString()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae055622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
index c12902c..9c72f72 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
@@ -440,6 +440,19 @@ public class TestFrameworkUploader {
}
Assert.assertTrue(uploader.checkSymlink(symlinkToTarget));
+ // Create a symlink to the target with /./ in the path
+ symlinkToTarget = new File(parent.getAbsolutePath() +
+ "/./symlinkToTarget2.txt");
+ try {
+ Files.createSymbolicLink(
+ Paths.get(symlinkToTarget.getAbsolutePath()),
+ Paths.get(targetFile.getAbsolutePath()));
+ } catch (UnsupportedOperationException e) {
+ // Symlinks are not supported, so ignore the test
+ Assume.assumeTrue(false);
+ }
+ Assert.assertTrue(uploader.checkSymlink(symlinkToTarget));
+
// Create a symlink outside the current directory
File symlinkOutside = new File(parent, "symlinkToParent.txt");
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: HDDS-192:Create new SCMCommand to
request a replication of a container. Contributed by Elek Marton
Posted by xk...@apache.org.
HDDS-192:Create new SCMCommand to request a replication of a container. Contributed by Elek Marton
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/238fe00a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/238fe00a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/238fe00a
Branch: refs/heads/HDFS-12943
Commit: 238fe00ad2692154f6a382f35735169ee5e4af2c
Parents: 35ec940
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Mon Jun 25 21:12:05 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Mon Jun 25 21:12:05 2018 -0700
----------------------------------------------------------------------
.../statemachine/DatanodeStateMachine.java | 3 +
.../ReplicateContainerCommandHandler.java | 67 ++++++++++++++
.../states/endpoint/HeartbeatEndpointTask.java | 12 +++
.../commands/ReplicateContainerCommand.java | 94 ++++++++++++++++++++
.../StorageContainerDatanodeProtocol.proto | 12 ++-
.../scm/server/SCMDatanodeProtocolServer.java | 11 +++
.../TestReplicateContainerHandler.java | 71 +++++++++++++++
7 files changed, 269 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index dc4e673..b073d7b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
.CommandDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
.DeleteBlocksCommandHandler;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
+ .ReplicateContainerCommandHandler;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
@@ -95,6 +97,7 @@ public class DatanodeStateMachine implements Closeable {
.addHandler(new CloseContainerCommandHandler())
.addHandler(new DeleteBlocksCommandHandler(
container.getContainerManager(), conf))
+ .addHandler(new ReplicateContainerCommandHandler())
.setConnectionManager(connectionManager)
.setContainer(container)
.setContext(context)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
new file mode 100644
index 0000000..b4e83b7
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.ozone.container.common.statemachine
+ .SCMConnectionManager;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Command handler to copy containers from sources.
+ */
+public class ReplicateContainerCommandHandler implements CommandHandler {
+ static final Logger LOG =
+ LoggerFactory.getLogger(ReplicateContainerCommandHandler.class);
+
+ private int invocationCount;
+
+ private long totalTime;
+
+ @Override
+ public void handle(SCMCommand command, OzoneContainer container,
+ StateContext context, SCMConnectionManager connectionManager) {
+ LOG.warn("Replicate command is not yet handled");
+
+ }
+
+ @Override
+ public SCMCommandProto.Type getCommandType() {
+ return Type.replicateContainerCommand;
+ }
+
+ @Override
+ public int getInvocationCount() {
+ return this.invocationCount;
+ }
+
+ @Override
+ public long getAverageRunTime() {
+ if (invocationCount > 0) {
+ return totalTime / invocationCount;
+ }
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
index 1ee6375..260a245 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.ozone.container.common.statemachine
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -196,6 +198,16 @@ public class HeartbeatEndpointTask
}
this.context.addCommand(closeContainer);
break;
+ case replicateContainerCommand:
+ ReplicateContainerCommand replicateContainerCommand =
+ ReplicateContainerCommand.getFromProtobuf(
+ commandResponseProto.getReplicateContainerCommandProto());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received SCM container replicate request for container {}",
+ replicateContainerCommand.getContainerID());
+ }
+ this.context.addCommand(replicateContainerCommand);
+ break;
default:
throw new IllegalArgumentException("Unknown response : "
+ commandResponseProto.getCommandType().name());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java
new file mode 100644
index 0000000..834318b
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.protocol.commands;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto
+ .Builder;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * SCM command to request replication of a container.
+ */
+public class ReplicateContainerCommand
+ extends SCMCommand<ReplicateContainerCommandProto> {
+
+ private final long containerID;
+
+ private final List<DatanodeDetails> sourceDatanodes;
+
+ public ReplicateContainerCommand(long containerID,
+ List<DatanodeDetails> sourceDatanodes) {
+ this.containerID = containerID;
+ this.sourceDatanodes = sourceDatanodes;
+ }
+
+ @Override
+ public Type getType() {
+ return SCMCommandProto.Type.replicateContainerCommand;
+ }
+
+ @Override
+ public byte[] getProtoBufMessage() {
+ return getProto().toByteArray();
+ }
+
+ public ReplicateContainerCommandProto getProto() {
+ Builder builder = ReplicateContainerCommandProto.newBuilder()
+ .setContainerID(containerID);
+ for (DatanodeDetails dd : sourceDatanodes) {
+ builder.addSources(dd.getProtoBufMessage());
+ }
+ return builder.build();
+ }
+
+ public static ReplicateContainerCommand getFromProtobuf(
+ ReplicateContainerCommandProto protoMessage) {
+ Preconditions.checkNotNull(protoMessage);
+
+ List<DatanodeDetails> datanodeDetails =
+ protoMessage.getSourcesList()
+ .stream()
+ .map(DatanodeDetails::getFromProtoBuf)
+ .collect(Collectors.toList());
+
+ return new ReplicateContainerCommand(protoMessage.getContainerID(),
+ datanodeDetails);
+
+ }
+
+ public long getContainerID() {
+ return containerID;
+ }
+
+ public List<DatanodeDetails> getSourceDatanodes() {
+ return sourceDatanodes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index f6aba05..54230c1 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -172,6 +172,7 @@ message SCMCommandProto {
deleteBlocksCommand = 2;
closeContainerCommand = 3;
deleteContainerCommand = 4;
+ replicateContainerCommand = 5;
}
// TODO: once we start using protoc 3.x, refactor this message using "oneof"
required Type commandType = 1;
@@ -179,6 +180,7 @@ message SCMCommandProto {
optional DeleteBlocksCommandProto deleteBlocksCommandProto = 3;
optional CloseContainerCommandProto closeContainerCommandProto = 4;
optional DeleteContainerCommandProto deleteContainerCommandProto = 5;
+ optional ReplicateContainerCommandProto replicateContainerCommandProto = 6;
}
/**
@@ -227,13 +229,21 @@ message CloseContainerCommandProto {
}
/**
-This command asks the datanode to close a specific container.
+This command asks the datanode to delete a specific container.
*/
message DeleteContainerCommandProto {
required int64 containerID = 1;
}
/**
+This command asks the datanode to replicate a container from specific sources.
+*/
+message ReplicateContainerCommandProto {
+ required int64 containerID = 1;
+ repeated DatanodeDetailsProto sources = 2;
+}
+
+/**
* Protocol used from a datanode to StorageContainerManager.
*
* Please see the request and response messages for details of the RPC calls.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index 7d16161..eb5ce1a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.protocol.proto
@@ -63,6 +64,9 @@ import static org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto
.Type.deleteBlocksCommand;
import static org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type
+ .replicateContainerCommand;
+import static org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto
.Type.reregisterCommand;
@@ -77,6 +81,7 @@ import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
import org.apache.hadoop.ozone.protocolPB
@@ -293,6 +298,12 @@ public class SCMDatanodeProtocolServer implements
.setCloseContainerCommandProto(
((CloseContainerCommand) cmd).getProto())
.build();
+ case replicateContainerCommand:
+ return builder
+ .setCommandType(replicateContainerCommand)
+ .setReplicateContainerCommandProto(
+ ((ReplicateContainerCommand)cmd).getProto())
+ .build();
default:
throw new IllegalArgumentException("Not implemented");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java
new file mode 100644
index 0000000..a5b101f
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.rest.OzoneException;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_CONTAINER_SIZE_GB;
+import org.junit.Test;
+
+/**
+ * Tests the behavior of the datanode, when replicate container command is
+ * received.
+ */
+public class TestReplicateContainerHandler {
+
+ @Test
+ public void test() throws IOException, TimeoutException, InterruptedException,
+ OzoneException {
+
+ GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
+ .captureLogs(ReplicateContainerCommandHandler.LOG);
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OZONE_SCM_CONTAINER_SIZE_GB, "1");
+ MiniOzoneCluster cluster =
+ MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
+ cluster.waitForClusterToBeReady();
+
+ DatanodeDetails datanodeDetails =
+ cluster.getHddsDatanodes().get(0).getDatanodeDetails();
+ //send the order to replicate the container
+ cluster.getStorageContainerManager().getScmNodeManager()
+ .addDatanodeCommand(datanodeDetails.getUuid(),
+ new ReplicateContainerCommand(1L,
+ new ArrayList<>()));
+
+ //TODO: here we test only the serialization/unserialization as
+ // the implementation is not yet done
+ GenericTestUtils
+ .waitFor(() -> logCapturer.getOutput().contains("not yet handled"), 500,
+ 5 * 1000);
+
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-8445. Improved error message
for duplicated service and component names. Contributed by Chandni Singh
Posted by xk...@apache.org.
YARN-8445. Improved error message for duplicated service and component names.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f15483c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f15483c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f15483c
Branch: refs/heads/HDFS-12943
Commit: 9f15483c5d7c94251f4c84e0155449188f202779
Parents: 43541a1
Author: Eric Yang <ey...@apache.org>
Authored: Thu Jun 21 11:18:14 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Jun 21 11:20:24 2018 -0400
----------------------------------------------------------------------
.../service/exceptions/RestApiErrorMessages.java | 2 ++
.../hadoop/yarn/service/utils/ServiceApiUtil.java | 5 +++++
.../hadoop/yarn/service/TestServiceApiUtil.java | 18 ++++++++++++++++++
3 files changed, 25 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f15483c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
index 1d2d719..5b3c72c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
@@ -28,6 +28,8 @@ public interface RestApiErrorMessages {
"than 63 characters";
String ERROR_COMPONENT_NAME_INVALID =
"Component name must be no more than %s characters: %s";
+ String ERROR_COMPONENT_NAME_CONFLICTS_WITH_SERVICE_NAME =
+ "Component name %s must not be same as service name %s";
String ERROR_USER_NAME_INVALID =
"User name must be no more than 63 characters";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f15483c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index 5499273..705e040 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -143,6 +143,11 @@ public class ServiceApiUtil {
throw new IllegalArgumentException(String.format(RestApiErrorMessages
.ERROR_COMPONENT_NAME_INVALID, maxCompLength, comp.getName()));
}
+ if (service.getName().equals(comp.getName())) {
+ throw new IllegalArgumentException(String.format(RestApiErrorMessages
+ .ERROR_COMPONENT_NAME_CONFLICTS_WITH_SERVICE_NAME,
+ comp.getName(), service.getName()));
+ }
if (componentNames.contains(comp.getName())) {
throw new IllegalArgumentException("Component name collision: " +
comp.getName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f15483c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
index 243c6b3..ae031d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
@@ -334,6 +334,24 @@ public class TestServiceApiUtil {
}
@Test
+ public void testComponentNameSameAsServiceName() throws IOException {
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs();
+ Service app = new Service();
+ app.setName("test");
+ app.setVersion("v1");
+ app.addComponent(createValidComponent("test"));
+
+ //component name same as service name
+ try {
+ ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+ Assert.fail(EXCEPTION_PREFIX + "component name matches service name");
+ } catch (IllegalArgumentException e) {
+ assertEquals("Component name test must not be same as service name test",
+ e.getMessage());
+ }
+ }
+
+ @Test
public void testExternalDuplicateComponent() throws IOException {
Service ext = createValidApplication("comp1");
SliderFileSystem sfs = ServiceTestUtils.initMockFs(ext);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HADOOP-15527. Improve delay check
for stopping processes. Fixed script location. Contributed by Vinod Kumar
Vavilapalli
Posted by xk...@apache.org.
HADOOP-15527. Improve delay check for stopping processes.
Fixed script location.
Contributed by Vinod Kumar Vavilapalli
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d87592f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d87592f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d87592f
Branch: refs/heads/HDFS-12943
Commit: 2d87592fc6a56bfe77dd3c11953caea2b701c846
Parents: f386e78
Author: Eric Yang <ey...@apache.org>
Authored: Tue Jun 19 13:38:13 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Tue Jun 19 13:38:13 2018 -0400
----------------------------------------------------------------------
.../test/scripts/process_with_sigterm_trap.sh | 24 ++++++++++++++++++++
.../test/scripts/process_with_sigterm_trap.sh | 24 --------------------
2 files changed, 24 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d87592f/hadoop-common-project/hadoop-common/src/test/scripts/process_with_sigterm_trap.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/process_with_sigterm_trap.sh b/hadoop-common-project/hadoop-common/src/test/scripts/process_with_sigterm_trap.sh
new file mode 100644
index 0000000..d7c7427
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/process_with_sigterm_trap.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+trap "echo SIGTERM trapped!" SIGTERM
+trap "echo SIGINT trapped!" SIGINT
+
+echo "$$" > "$1"
+
+while true; do
+ sleep 1.3
+done
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d87592f/hadoop-common/src/test/scripts/process_with_sigterm_trap.sh
----------------------------------------------------------------------
diff --git a/hadoop-common/src/test/scripts/process_with_sigterm_trap.sh b/hadoop-common/src/test/scripts/process_with_sigterm_trap.sh
deleted file mode 100644
index d7c7427..0000000
--- a/hadoop-common/src/test/scripts/process_with_sigterm_trap.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-trap "echo SIGTERM trapped!" SIGTERM
-trap "echo SIGINT trapped!" SIGINT
-
-echo "$$" > "$1"
-
-while true; do
- sleep 1.3
-done
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HDDS-177. Create a releasable
ozonefs artifact Contributed by Marton, Elek.
Posted by xk...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
new file mode 100644
index 0000000..3156eb2
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract test for ROOT directory operations.
+ */
+public class ITestOzoneContractRootDir extends
+ AbstractContractRootDirectoryTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
new file mode 100644
index 0000000..c4bc0ff
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests covering file seek.
+ */
+public class ITestOzoneContractSeek extends AbstractContractSeekTest {
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
new file mode 100644
index 0000000..8417e46
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.rest.OzoneException;
+import org.apache.hadoop.ozone.web.handlers.BucketArgs;
+import org.apache.hadoop.ozone.web.handlers.UserArgs;
+import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
+import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.junit.Assert;
+
+import java.io.IOException;
+
+/**
+ * The contract of Ozone: only enabled if the test bucket is provided.
+ */
+class OzoneContract extends AbstractFSContract {
+
+ private static MiniOzoneCluster cluster;
+ private static StorageHandler storageHandler;
+ private static final String CONTRACT_XML = "contract/ozone.xml";
+
+ OzoneContract(Configuration conf) {
+ super(conf);
+ //insert the base features
+ addConfResource(CONTRACT_XML);
+ }
+
+ @Override
+ public String getScheme() {
+ return OzoneConsts.OZONE_URI_SCHEME;
+ }
+
+ @Override
+ public Path getTestPath() {
+ Path path = new Path("/test");
+ return path;
+ }
+
+ public static void createCluster() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.addResource(CONTRACT_XML);
+
+ cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
+ try {
+ cluster.waitForClusterToBeReady();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
+ }
+
+ private void copyClusterConfigs(String configKey) {
+ getConf().set(configKey, cluster.getConf().get(configKey));
+ }
+
+ @Override
+ public FileSystem getTestFileSystem() throws IOException {
+ //assumes cluster is not null
+ Assert.assertNotNull("cluster not created", cluster);
+
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+
+
+ UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
+ null, null, null, null);
+ VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
+ volumeArgs.setUserName(userName);
+ volumeArgs.setAdminName(adminName);
+ BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
+ try {
+ storageHandler.createVolume(volumeArgs);
+ storageHandler.createBucket(bucketArgs);
+ } catch (OzoneException e) {
+ throw new IOException(e.getMessage());
+ }
+
+ String uri = String.format("%s://%s.%s/",
+ OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
+ getConf().set("fs.defaultFS", uri);
+ copyClusterConfigs(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY);
+ copyClusterConfigs(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
+ return FileSystem.get(getConf());
+ }
+
+ public static void destroyCluster() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/resources/contract/ozone.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/resources/contract/ozone.xml b/hadoop-ozone/ozonefs/src/test/resources/contract/ozone.xml
new file mode 100644
index 0000000..fe2075c
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/resources/contract/ozone.xml
@@ -0,0 +1,113 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <!--
+ Ozone is a blobstore, with very different behavior than a classic filesystem.
+ -->
+
+ <property>
+ <name>fs.contract.test.root-tests-enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.test.random-seek-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-blobstore</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.create-visibility-delayed</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-case-sensitive</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-returns-false-if-source-missing</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-remove-dest-if-empty-dir</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-append</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-directory-delete</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-rename</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-block-locality</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-concat</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-getfilestatus</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek-on-closed-file</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rejects-seek-past-eof</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-strict-exceptions</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-unix-permissions</name>
+ <value>false</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/resources/log4j.properties b/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
new file mode 100644
index 0000000..3bf1619
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
@@ -0,0 +1,23 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+
+# for debugging low level Ozone operations, uncomment this line
+# log4j.logger.org.apache.hadoop.ozone=DEBUG
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index cffef14..b655088 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -34,6 +34,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<module>client</module>
<module>ozone-manager</module>
<module>tools</module>
+ <module>ozonefs</module>
<module>integration-test</module>
<module>objectstore-service</module>
<module>docs</module>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index ed0187b..dfd1eac 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -586,7 +586,11 @@
<artifactId>hadoop-ozone-docs</artifactId>
<version>${hdds.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem</artifactId>
+ <version>${hdds.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/pom.xml b/hadoop-tools/hadoop-ozone/pom.xml
deleted file mode 100644
index a7d0cfa..0000000
--- a/hadoop-tools/hadoop-ozone/pom.xml
+++ /dev/null
@@ -1,174 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
- <relativePath>../../hadoop-project</relativePath>
- </parent>
- <artifactId>hadoop-ozone-filesystem</artifactId>
- <name>Apache Hadoop Ozone FileSystem</name>
- <packaging>jar</packaging>
-
- <properties>
- <file.encoding>UTF-8</file.encoding>
- <downloadSources>true</downloadSources>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>deplist</id>
- <phase>compile</phase>
- <goals>
- <goal>list</goal>
- </goals>
- <configuration>
- <!-- build a shellprofile -->
- <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-server-scm</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-server-framework</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-ozone-manager</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-container-service</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-client</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-objectstore-service</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-integration-test</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
deleted file mode 100644
index 832a0cb..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-/**
- * Constants for Ozone FileSystem implementation.
- */
-public final class Constants {
-
- public static final String OZONE_DEFAULT_USER = "hdfs";
-
- public static final String OZONE_USER_DIR = "/user";
-
- /** Local buffer directory. */
- public static final String BUFFER_DIR_KEY = "fs.ozone.buffer.dir";
-
- /** Temporary directory. */
- public static final String BUFFER_TMP_KEY = "hadoop.tmp.dir";
-
- /** Page size for Ozone listing operation. */
- public static final int LISTING_PAGE_SIZE = 1024;
-
- private Constants() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
deleted file mode 100644
index 4163c13..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.DelegateToFileSystem;
-import org.apache.hadoop.ozone.OzoneConsts;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * ozone implementation of AbstractFileSystem.
- * This impl delegates to the OzoneFileSystem
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class OzFs extends DelegateToFileSystem {
-
- public OzFs(URI theUri, Configuration conf)
- throws IOException, URISyntaxException {
- super(theUri, new OzoneFileSystem(), conf,
- OzoneConsts.OZONE_URI_SCHEME, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
deleted file mode 100644
index 4c5c0c8..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * The input stream for Ozone file system.
- *
- * TODO: Make inputStream generic for both rest and rpc clients
- * This class is not thread safe.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public final class OzoneFSInputStream extends FSInputStream {
-
- private final ChunkGroupInputStream inputStream;
-
- public OzoneFSInputStream(InputStream inputStream) {
- this.inputStream = (ChunkGroupInputStream)inputStream;
- }
-
- @Override
- public int read() throws IOException {
- return inputStream.read();
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- return inputStream.read(b, off, len);
- }
-
- @Override
- public synchronized void close() throws IOException {
- inputStream.close();
- }
-
- @Override
- public void seek(long pos) throws IOException {
- inputStream.seek(pos);
- }
-
- @Override
- public long getPos() throws IOException {
- return inputStream.getPos();
- }
-
- @Override
- public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
-
- @Override
- public int available() throws IOException {
- return inputStream.available();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
deleted file mode 100644
index faa3628..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
-
-
-/**
- * The output stream for Ozone file system.
- *
- * TODO: Make outputStream generic for both rest and rpc clients
- * This class is not thread safe.
- */
-public class OzoneFSOutputStream extends OutputStream {
-
- private final ChunkGroupOutputStream outputStream;
-
- public OzoneFSOutputStream(OutputStream outputStream) {
- this.outputStream = (ChunkGroupOutputStream)outputStream;
- }
-
- @Override
- public void write(int b) throws IOException {
- outputStream.write(b);
- }
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- outputStream.write(b, off, len);
- }
-
- @Override
- public synchronized void flush() throws IOException {
- outputStream.flush();
- }
-
- @Override
- public synchronized void close() throws IOException {
- outputStream.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
deleted file mode 100644
index 6906a9d..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++ /dev/null
@@ -1,689 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Iterator;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.client.ObjectStore;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientFactory;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-
-import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
-import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
-import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
-
-/**
- * The Ozone Filesystem implementation.
- *
- * This subclass is marked as private as code should not be creating it
- * directly; use {@link FileSystem#get(Configuration)} and variants to create
- * one. If cast to {@link OzoneFileSystem}, extra methods and features may be
- * accessed. Consider those private and unstable.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class OzoneFileSystem extends FileSystem {
- static final Logger LOG = LoggerFactory.getLogger(OzoneFileSystem.class);
-
- /** The Ozone client for connecting to Ozone server. */
- private OzoneClient ozoneClient;
- private ObjectStore objectStore;
- private OzoneVolume volume;
- private OzoneBucket bucket;
- private URI uri;
- private String userName;
- private Path workingDir;
- private ReplicationType replicationType;
- private ReplicationFactor replicationFactor;
-
- private static final Pattern URL_SCHEMA_PATTERN =
- Pattern.compile("(.+)\\.([^\\.]+)");
-
- @Override
- public void initialize(URI name, Configuration conf) throws IOException {
- super.initialize(name, conf);
- setConf(conf);
- Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
- assert getScheme().equals(name.getScheme());
-
- String authority = name.getAuthority();
-
- Matcher matcher = URL_SCHEMA_PATTERN.matcher(authority);
-
- if (!matcher.matches()) {
- throw new IllegalArgumentException("Ozone file system url should be "
- + "in the form o3://bucket.volume");
- }
- String bucketStr = matcher.group(1);
- String volumeStr = matcher.group(2);
-
- try {
- uri = new URIBuilder().setScheme(OZONE_URI_SCHEME)
- .setHost(authority).build();
- LOG.trace("Ozone URI for ozfs initialization is " + uri);
- this.ozoneClient = OzoneClientFactory.getRpcClient(conf);
- objectStore = ozoneClient.getObjectStore();
- this.volume = objectStore.getVolume(volumeStr);
- this.bucket = volume.getBucket(bucketStr);
- this.replicationType = ReplicationType.valueOf(
- conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
- OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT));
- this.replicationFactor = ReplicationFactor.valueOf(
- conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
- OzoneConfigKeys.OZONE_REPLICATION_DEFAULT));
- try {
- this.userName =
- UserGroupInformation.getCurrentUser().getShortUserName();
- } catch (IOException e) {
- this.userName = OZONE_DEFAULT_USER;
- }
- this.workingDir = new Path(OZONE_USER_DIR, this.userName)
- .makeQualified(this.uri, this.workingDir);
- } catch (URISyntaxException ue) {
- final String msg = "Invalid Ozone endpoint " + name;
- LOG.error(msg, ue);
- throw new IOException(msg, ue);
- }
- }
-
- @Override
- public void close() throws IOException {
- try {
- ozoneClient.close();
- } finally {
- super.close();
- }
- }
-
- @Override
- public URI getUri() {
- return uri;
- }
-
- @Override
- public String getScheme() {
- return OZONE_URI_SCHEME;
- }
-
- @Override
- public FSDataInputStream open(Path f, int bufferSize) throws IOException {
- LOG.trace("open() path:{}", f);
- final FileStatus fileStatus = getFileStatus(f);
- final String key = pathToKey(f);
- if (fileStatus.isDirectory()) {
- throw new FileNotFoundException("Can't open directory " + f + " to read");
- }
-
- return new FSDataInputStream(
- new OzoneFSInputStream(bucket.readKey(key).getInputStream()));
- }
-
- @Override
- public FSDataOutputStream create(Path f, FsPermission permission,
- boolean overwrite, int bufferSize,
- short replication, long blockSize,
- Progressable progress) throws IOException {
- LOG.trace("create() path:{}", f);
- final String key = pathToKey(f);
- final FileStatus status;
- try {
- status = getFileStatus(f);
- if (status.isDirectory()) {
- throw new FileAlreadyExistsException(f + " is a directory");
- } else {
- if (!overwrite) {
- // path references a file and overwrite is disabled
- throw new FileAlreadyExistsException(f + " already exists");
- }
- LOG.trace("Overwriting file {}", f);
- deleteObject(key);
- }
- } catch (FileNotFoundException ignored) {
- // check if the parent directory needs to be created
- Path parent = f.getParent();
- try {
- // create all the directories for the parent
- FileStatus parentStatus = getFileStatus(parent);
- LOG.trace("parent key:{} status:{}", key, parentStatus);
- } catch (FileNotFoundException e) {
- mkdirs(parent);
- }
- // This exception needs to ignored as this means that the file currently
- // does not exists and a new file can thus be created.
- }
-
- OzoneOutputStream ozoneOutputStream =
- bucket.createKey(key, 0, replicationType, replicationFactor);
- // We pass null to FSDataOutputStream so it won't count writes that
- // are being buffered to a file
- return new FSDataOutputStream(
- new OzoneFSOutputStream(ozoneOutputStream.getOutputStream()), null);
- }
-
- @Override
- public FSDataOutputStream createNonRecursive(Path path,
- FsPermission permission,
- EnumSet<CreateFlag> flags,
- int bufferSize,
- short replication,
- long blockSize,
- Progressable progress) throws IOException {
- final Path parent = path.getParent();
- if (parent != null) {
- // expect this to raise an exception if there is no parent
- if (!getFileStatus(parent).isDirectory()) {
- throw new FileAlreadyExistsException("Not a directory: " + parent);
- }
- }
- return create(path, permission, flags.contains(CreateFlag.OVERWRITE),
- bufferSize, replication, blockSize, progress);
- }
-
- @Override
- public FSDataOutputStream append(Path f, int bufferSize,
- Progressable progress) throws IOException {
- throw new UnsupportedOperationException("append() Not implemented by the "
- + getClass().getSimpleName() + " FileSystem implementation");
- }
-
- private class RenameIterator extends OzoneListingIterator {
- private final String srcKey;
- private final String dstKey;
-
- RenameIterator(Path srcPath, Path dstPath)
- throws IOException {
- super(srcPath);
- srcKey = pathToKey(srcPath);
- dstKey = pathToKey(dstPath);
- LOG.trace("rename from:{} to:{}", srcKey, dstKey);
- }
-
- boolean processKey(String key) throws IOException {
- String newKeyName = dstKey.concat(key.substring(srcKey.length()));
- bucket.renameKey(key, newKeyName);
- return true;
- }
- }
-
- /**
- * Check whether the source and destination path are valid and then perform
- * rename from source path to destination path.
- *
- * The rename operation is performed by renaming the keys with src as prefix.
- * For such keys the prefix is changed from src to dst.
- *
- * @param src source path for rename
- * @param dst destination path for rename
- * @return true if rename operation succeeded or
- * if the src and dst have the same path and are of the same type
- * @throws IOException on I/O errors or if the src/dst paths are invalid.
- */
- @Override
- public boolean rename(Path src, Path dst) throws IOException {
- if (src.equals(dst)) {
- return true;
- }
-
- LOG.trace("rename() from:{} to:{}", src, dst);
- if (src.isRoot()) {
- // Cannot rename root of file system
- LOG.trace("Cannot rename the root of a filesystem");
- return false;
- }
-
- // Cannot rename a directory to its own subdirectory
- Path dstParent = dst.getParent();
- while (dstParent != null && !src.equals(dstParent)) {
- dstParent = dstParent.getParent();
- }
- Preconditions.checkArgument(dstParent == null,
- "Cannot rename a directory to its own subdirectory");
- // Check if the source exists
- FileStatus srcStatus;
- try {
- srcStatus = getFileStatus(src);
- } catch (FileNotFoundException fnfe) {
- // source doesn't exist, return
- return false;
- }
-
- // Check if the destination exists
- FileStatus dstStatus;
- try {
- dstStatus = getFileStatus(dst);
- } catch (FileNotFoundException fnde) {
- dstStatus = null;
- }
-
- if (dstStatus == null) {
- // If dst doesn't exist, check whether dst parent dir exists or not
- // if the parent exists, the source can still be renamed to dst path
- dstStatus = getFileStatus(dst.getParent());
- if (!dstStatus.isDirectory()) {
- throw new IOException(String.format(
- "Failed to rename %s to %s, %s is a file", src, dst,
- dst.getParent()));
- }
- } else {
- // if dst exists and source and destination are same,
- // check both the src and dst are of same type
- if (srcStatus.getPath().equals(dstStatus.getPath())) {
- return !srcStatus.isDirectory();
- } else if (dstStatus.isDirectory()) {
- // If dst is a directory, rename source as subpath of it.
- // for example rename /source to /dst will lead to /dst/source
- dst = new Path(dst, src.getName());
- FileStatus[] statuses;
- try {
- statuses = listStatus(dst);
- } catch (FileNotFoundException fnde) {
- statuses = null;
- }
-
- if (statuses != null && statuses.length > 0) {
- // If dst exists and not a directory not empty
- throw new FileAlreadyExistsException(String.format(
- "Failed to rename %s to %s, file already exists or not empty!",
- src, dst));
- }
- } else {
- // If dst is not a directory
- throw new FileAlreadyExistsException(String.format(
- "Failed to rename %s to %s, file already exists!", src, dst));
- }
- }
-
- if (srcStatus.isDirectory()) {
- if (dst.toString().startsWith(src.toString())) {
- LOG.trace("Cannot rename a directory to a subdirectory of self");
- return false;
- }
- }
- RenameIterator iterator = new RenameIterator(src, dst);
- return iterator.iterate();
- }
-
- private class DeleteIterator extends OzoneListingIterator {
- private boolean recursive;
- DeleteIterator(Path f, boolean recursive)
- throws IOException {
- super(f);
- this.recursive = recursive;
- if (getStatus().isDirectory()
- && !this.recursive
- && listStatus(f).length != 0) {
- throw new PathIsNotEmptyDirectoryException(f.toString());
- }
- }
-
- boolean processKey(String key) throws IOException {
- if (key.equals("")) {
- LOG.trace("Skipping deleting root directory");
- return true;
- } else {
- LOG.trace("deleting key:" + key);
- boolean succeed = deleteObject(key);
- // if recursive delete is requested ignore the return value of
- // deleteObject and issue deletes for other keys.
- return recursive || succeed;
- }
- }
- }
-
- @Override
- public boolean delete(Path f, boolean recursive) throws IOException {
- LOG.trace("delete() path:{} recursive:{}", f, recursive);
- try {
- DeleteIterator iterator = new DeleteIterator(f, recursive);
- return iterator.iterate();
- } catch (FileNotFoundException e) {
- LOG.debug("Couldn't delete {} - does not exist", f);
- return false;
- }
- }
-
- private class ListStatusIterator extends OzoneListingIterator {
- private List<FileStatus> statuses = new ArrayList<>(LISTING_PAGE_SIZE);
- private Path f;
-
- ListStatusIterator(Path f) throws IOException {
- super(f);
- this.f = f;
- }
-
- boolean processKey(String key) throws IOException {
- Path keyPath = new Path(OZONE_URI_DELIMITER + key);
- if (key.equals(getPathKey())) {
- if (pathIsDirectory()) {
- return true;
- } else {
- statuses.add(getFileStatus(keyPath));
- return true;
- }
- }
- // left with only subkeys now
- if (pathToKey(keyPath.getParent()).equals(pathToKey(f))) {
- // skip keys which are for subdirectories of the directory
- statuses.add(getFileStatus(keyPath));
- }
- return true;
- }
-
- FileStatus[] getStatuses() {
- return statuses.toArray(new FileStatus[statuses.size()]);
- }
- }
-
- @Override
- public FileStatus[] listStatus(Path f) throws IOException {
- LOG.trace("listStatus() path:{}", f);
- ListStatusIterator iterator = new ListStatusIterator(f);
- iterator.iterate();
- return iterator.getStatuses();
- }
-
- @Override
- public void setWorkingDirectory(Path newDir) {
- workingDir = newDir;
- }
-
- @Override
- public Path getWorkingDirectory() {
- return workingDir;
- }
-
- /**
- * Check whether the path is valid and then create directories.
- * Directory is represented using a key with no value.
- * All the non-existent parent directories are also created.
- *
- * @param path directory path to be created
- * @return true if directory exists or created successfully.
- * @throws IOException
- */
- private boolean mkdir(Path path) throws IOException {
- Path fPart = path;
- Path prevfPart = null;
- do {
- LOG.trace("validating path:{}", fPart);
- try {
- FileStatus fileStatus = getFileStatus(fPart);
- if (fileStatus.isDirectory()) {
- // If path exists and a directory, exit
- break;
- } else {
- // Found a file here, rollback and delete newly created directories
- LOG.trace("Found a file with same name as directory, path:{}", fPart);
- if (prevfPart != null) {
- delete(prevfPart, true);
- }
- throw new FileAlreadyExistsException(String.format(
- "Can't make directory for path '%s', it is a file.", fPart));
- }
- } catch (FileNotFoundException fnfe) {
- LOG.trace("creating directory for fpart:{}", fPart);
- String key = pathToKey(fPart);
- String dirKey = addTrailingSlashIfNeeded(key);
- if (!createDirectory(dirKey)) {
- // Directory creation failed here,
- // rollback and delete newly created directories
- LOG.trace("Directory creation failed, path:{}", fPart);
- if (prevfPart != null) {
- delete(prevfPart, true);
- }
- return false;
- }
- }
- prevfPart = fPart;
- fPart = fPart.getParent();
- } while (fPart != null);
- return true;
- }
-
- @Override
- public boolean mkdirs(Path f, FsPermission permission) throws IOException {
- LOG.trace("mkdir() path:{} ", f);
- String key = pathToKey(f);
- if (StringUtils.isEmpty(key)) {
- return false;
- }
- return mkdir(f);
- }
-
- @Override
- public FileStatus getFileStatus(Path f) throws IOException {
- LOG.trace("getFileStatus() path:{}", f);
- Path qualifiedPath = f.makeQualified(uri, workingDir);
- String key = pathToKey(qualifiedPath);
-
- if (key.length() == 0) {
- return new FileStatus(0, true, 1, 0,
- bucket.getCreationTime(), qualifiedPath);
- }
-
- // consider this a file and get key status
- OzoneKey meta = getKeyInfo(key);
- if (meta == null) {
- key = addTrailingSlashIfNeeded(key);
- meta = getKeyInfo(key);
- }
-
- if (meta == null) {
- LOG.trace("File:{} not found", f);
- throw new FileNotFoundException(f + ": No such file or directory!");
- } else if (isDirectory(meta)) {
- return new FileStatus(0, true, 1, 0,
- meta.getModificationTime(), qualifiedPath);
- } else {
- //TODO: Fetch replication count from ratis config
- return new FileStatus(meta.getDataSize(), false, 1,
- getDefaultBlockSize(f), meta.getModificationTime(), qualifiedPath);
- }
- }
-
- /**
- * Helper method to fetch the key metadata info.
- * @param key key whose metadata information needs to be fetched
- * @return metadata info of the key
- */
- private OzoneKey getKeyInfo(String key) {
- try {
- return bucket.getKey(key);
- } catch (IOException e) {
- LOG.trace("Key:{} does not exists", key);
- return null;
- }
- }
-
- /**
- * Helper method to check if an Ozone key is representing a directory.
- * @param key key to be checked as a directory
- * @return true if key is a directory, false otherwise
- */
- private boolean isDirectory(OzoneKey key) {
- LOG.trace("key name:{} size:{}", key.getName(),
- key.getDataSize());
- return key.getName().endsWith(OZONE_URI_DELIMITER)
- && (key.getDataSize() == 0);
- }
-
- /**
- * Helper method to create an directory specified by key name in bucket.
- * @param keyName key name to be created as directory
- * @return true if the key is created, false otherwise
- */
- private boolean createDirectory(String keyName) {
- try {
- LOG.trace("creating dir for key:{}", keyName);
- bucket.createKey(keyName, 0, replicationType, replicationFactor).close();
- return true;
- } catch (IOException ioe) {
- LOG.error("create key failed for key:{}", keyName, ioe);
- return false;
- }
- }
-
- /**
- * Helper method to delete an object specified by key name in bucket.
- * @param keyName key name to be deleted
- * @return true if the key is deleted, false otherwise
- */
- private boolean deleteObject(String keyName) {
- LOG.trace("issuing delete for key" + keyName);
- try {
- bucket.deleteKey(keyName);
- return true;
- } catch (IOException ioe) {
- LOG.error("delete key failed " + ioe.getMessage());
- return false;
- }
- }
-
- /**
- * Turn a path (relative or otherwise) into an Ozone key.
- *
- * @param path the path of the file.
- * @return the key of the object that represents the file.
- */
- public String pathToKey(Path path) {
- Objects.requireNonNull(path, "Path can not be null!");
- if (!path.isAbsolute()) {
- path = new Path(workingDir, path);
- }
- // removing leading '/' char
- String key = path.toUri().getPath().substring(1);
- LOG.trace("path for key:{} is:{}", key, path);
- return key;
- }
-
- /**
- * Add trailing delimiter to path if it is already not present.
- *
- * @param key the ozone Key which needs to be appended
- * @return delimiter appended key
- */
- private String addTrailingSlashIfNeeded(String key) {
- if (StringUtils.isNotEmpty(key) && !key.endsWith(OZONE_URI_DELIMITER)) {
- return key + OZONE_URI_DELIMITER;
- } else {
- return key;
- }
- }
-
- @Override
- public String toString() {
- return "OzoneFileSystem{URI=" + uri + ", "
- + "workingDir=" + workingDir + ", "
- + "userName=" + userName + ", "
- + "statistics=" + statistics
- + "}";
- }
-
- private abstract class OzoneListingIterator {
- private final Path path;
- private final FileStatus status;
- private String pathKey;
- private Iterator<OzoneKey> keyIterator;
-
- OzoneListingIterator(Path path)
- throws IOException {
- this.path = path;
- this.status = getFileStatus(path);
- this.pathKey = pathToKey(path);
- if (status.isDirectory()) {
- this.pathKey = addTrailingSlashIfNeeded(pathKey);
- }
- keyIterator = bucket.listKeys(pathKey);
- }
-
- abstract boolean processKey(String key) throws IOException;
-
- // iterates all the keys in the particular path
- boolean iterate() throws IOException {
- LOG.trace("Iterating path {}", path);
- if (status.isDirectory()) {
- LOG.trace("Iterating directory:{}", pathKey);
- while (keyIterator.hasNext()) {
- OzoneKey key = keyIterator.next();
- LOG.trace("iterating key:{}", key.getName());
- if (!processKey(key.getName())) {
- return false;
- }
- }
- return true;
- } else {
- LOG.trace("iterating file:{}", path);
- return processKey(pathKey);
- }
- }
-
- String getPathKey() {
- return pathKey;
- }
-
- boolean pathIsDirectory() {
- return status.isDirectory();
- }
-
- FileStatus getStatus() {
- return status;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
deleted file mode 100644
index 93e82c3..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * Ozone Filesystem.
- *
- * Except for the exceptions, it should all be hidden as implementation details.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
deleted file mode 100644
index ad21f28..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-/**
- * Test OzoneFSInputStream by reading through multiple interfaces.
- */
-public class TestOzoneFSInputStream {
- private static MiniOzoneCluster cluster = null;
- private static FileSystem fs;
- private static StorageHandler storageHandler;
- private static Path filePath = null;
- private static byte[] data = null;
-
- /**
- * Create a MiniDFSCluster for testing.
- * <p>
- * Ozone is made active by setting OZONE_ENABLED = true and
- * OZONE_HANDLER_TYPE_KEY = "distributed"
- *
- * @throws IOException
- */
- @BeforeClass
- public static void init() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 10);
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(10)
- .build();
- cluster.waitForClusterToBeReady();
- storageHandler =
- new ObjectStoreHandler(conf).getStorageHandler();
-
- // create a volume and a bucket to be used by OzoneFileSystem
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
-
- // Fetch the host and port for File System init
- DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0)
- .getDatanodeDetails();
- int port = datanodeDetails
- .getPort(DatanodeDetails.Port.Name.REST).getValue();
- String host = datanodeDetails.getHostName();
-
- // Set the fs.defaultFS and start the filesystem
- String uri = String.format("%s://%s.%s/",
- OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
- conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, uri);
- fs = FileSystem.get(conf);
- int fileLen = 100 * 1024 * 1024;
- data = DFSUtil.string2Bytes(RandomStringUtils.randomAlphanumeric(fileLen));
- filePath = new Path("/" + RandomStringUtils.randomAlphanumeric(5));
- try (FSDataOutputStream stream = fs.create(filePath)) {
- stream.write(data);
- }
- }
-
- /**
- * Shutdown MiniDFSCluster.
- */
- @AfterClass
- public static void shutdown() throws IOException {
- fs.close();
- storageHandler.close();
- cluster.shutdown();
- }
-
- @Test
- public void testO3FSSingleByteRead() throws IOException {
- FSDataInputStream inputStream = fs.open(filePath);
- byte[] value = new byte[data.length];
- int i = 0;
- while(true) {
- int val = inputStream.read();
- if (val == -1) {
- break;
- }
- value[i] = (byte)val;
- Assert.assertEquals("value mismatch at:" + i, value[i], data[i]);
- i++;
- }
- Assert.assertEquals(i, data.length);
- Assert.assertTrue(Arrays.equals(value, data));
- inputStream.close();
- }
-
- @Test
- public void testO3FSMultiByteRead() throws IOException {
- FSDataInputStream inputStream = fs.open(filePath);
- byte[] value = new byte[data.length];
- byte[] tmp = new byte[1* 1024 *1024];
- int i = 0;
- while(true) {
- int val = inputStream.read(tmp);
- if (val == -1) {
- break;
- }
- System.arraycopy(tmp, 0, value, i * tmp.length, tmp.length);
- i++;
- }
- Assert.assertEquals(i * tmp.length, data.length);
- Assert.assertTrue(Arrays.equals(value, data));
- inputStream.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
deleted file mode 100644
index a225702..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.junit.After;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Time;
-
-import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test OzoneFileSystem Interfaces.
- *
- * This test will test the various interfaces i.e.
- * create, read, write, getFileStatus
- */
-@RunWith(Parameterized.class)
-public class TestOzoneFileInterfaces {
-
- private String rootPath;
- private String userName;
-
- /**
- * Parameter class to set absolute url/defaultFS handling.
- * <p>
- * Hadoop file systems could be used in multiple ways: Using the defaultfs
- * and file path without the schema, or use absolute url-s even with
- * different defaultFS. This parameter matrix would test both the use cases.
- */
- @Parameters
- public static Collection<Object[]> data() {
- return Arrays.asList(new Object[][] {{false, true}, {true, false}});
- }
-
- private boolean setDefaultFs;
-
- private boolean useAbsolutePath;
-
- private static MiniOzoneCluster cluster = null;
-
- private static FileSystem fs;
-
- private static StorageHandler storageHandler;
-
- public TestOzoneFileInterfaces(boolean setDefaultFs,
- boolean useAbsolutePath) {
- this.setDefaultFs = setDefaultFs;
- this.useAbsolutePath = useAbsolutePath;
- }
-
- @Before
- public void init() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(3)
- .build();
- cluster.waitForClusterToBeReady();
- storageHandler =
- new ObjectStoreHandler(conf).getStorageHandler();
-
- // create a volume and a bucket to be used by OzoneFileSystem
- userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
-
- rootPath = String
- .format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucketName,
- volumeName);
- if (setDefaultFs) {
- // Set the fs.defaultFS and start the filesystem
- conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
- fs = FileSystem.get(conf);
- } else {
- fs = FileSystem.get(new URI(rootPath + "/test.txt"), conf);
- }
- }
-
- @After
- public void teardown() throws IOException {
- if (cluster != null) {
- cluster.shutdown();
- }
- IOUtils.closeQuietly(fs);
- IOUtils.closeQuietly(storageHandler);
- }
-
- @Test
- public void testFileSystemInit() throws IOException {
- if (setDefaultFs) {
- assertTrue(
- "The initialized file system is not OzoneFileSystem but " +
- fs.getClass(),
- fs instanceof OzoneFileSystem);
- assertEquals(OzoneConsts.OZONE_URI_SCHEME, fs.getUri().getScheme());
- }
- }
-
- @Test
- public void testOzFsReadWrite() throws IOException {
- long currentTime = Time.now();
- int stringLen = 20;
- String data = RandomStringUtils.randomAlphanumeric(stringLen);
- String filePath = RandomStringUtils.randomAlphanumeric(5);
- Path path = createPath("/" + filePath);
- try (FSDataOutputStream stream = fs.create(path)) {
- stream.writeBytes(data);
- }
-
- FileStatus status = fs.getFileStatus(path);
- // The timestamp of the newly created file should always be greater than
- // the time when the test was started
- assertTrue("Modification time has not been recorded: " + status,
- status.getModificationTime() > currentTime);
-
- try (FSDataInputStream inputStream = fs.open(path)) {
- byte[] buffer = new byte[stringLen];
- inputStream.readFully(0, buffer);
- String out = new String(buffer, 0, buffer.length);
- assertEquals(data, out);
- }
- }
-
-
- @Test
- public void testDirectory() throws IOException {
- String dirPath = RandomStringUtils.randomAlphanumeric(5);
- Path path = createPath("/" + dirPath);
- assertTrue("Makedirs returned with false for the path " + path,
- fs.mkdirs(path));
-
- FileStatus status = fs.getFileStatus(path);
- assertTrue("The created path is not directory.", status.isDirectory());
-
- assertEquals(0, status.getLen());
-
- FileStatus[] statusList = fs.listStatus(createPath("/"));
- assertEquals(1, statusList.length);
- assertEquals(status, statusList[0]);
-
- FileStatus statusRoot = fs.getFileStatus(createPath("/"));
- assertTrue("Root dir (/) is not a directory.", status.isDirectory());
- assertEquals(0, status.getLen());
-
-
- }
-
- @Test
- public void testPathToKey() throws Exception {
- OzoneFileSystem ozoneFs = (OzoneFileSystem) TestOzoneFileInterfaces.fs;
-
- assertEquals("a/b/1", ozoneFs.pathToKey(new Path("/a/b/1")));
-
- assertEquals("user/" + getCurrentUser() + "/key1/key2",
- ozoneFs.pathToKey(new Path("key1/key2")));
-
- assertEquals("key1/key2",
- ozoneFs.pathToKey(new Path("o3://test1/key1/key2")));
- }
-
- private String getCurrentUser() {
- try {
- return UserGroupInformation.getCurrentUser().getShortUserName();
- } catch (IOException e) {
- return OZONE_DEFAULT_USER;
- }
- }
-
- private Path createPath(String relativePath) {
- if (useAbsolutePath) {
- return new Path(
- rootPath + (relativePath.startsWith("/") ? "" : "/") + relativePath);
- } else {
- return new Path(relativePath);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
deleted file mode 100644
index dd54315..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests creating files.
- */
-public class ITestOzoneContractCreate extends AbstractContractCreateTest {
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
deleted file mode 100644
index f0a3d8d..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests covering deletes.
- */
-public class ITestOzoneContractDelete extends AbstractContractDeleteTest {
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
deleted file mode 100644
index 134a9ad..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-
-/**
- * Contract test suite covering S3A integration with DistCp.
- * Uses the block output stream, buffered to disk. This is the
- * recommended output mechanism for DistCP due to its scalability.
- */
-public class ITestOzoneContractDistCp extends AbstractContractDistCpTest {
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected OzoneContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
deleted file mode 100644
index 98bbb14..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests covering getFileStatus.
- */
-public class ITestOzoneContractGetFileStatus
- extends AbstractContractGetFileStatusTest {
-
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-
- @Override
- public void teardown() throws Exception {
- getLog().info("FS details {}", getFileSystem());
- super.teardown();
- }
-
- @Override
- protected Configuration createConfiguration() {
- return super.createConfiguration();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
deleted file mode 100644
index bc0de5d..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Test dir operations on Ozone.
- */
-public class ITestOzoneContractMkdir extends AbstractContractMkdirTest {
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
deleted file mode 100644
index 0bc57d4..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests opening files.
- */
-public class ITestOzoneContractOpen extends AbstractContractOpenTest {
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
deleted file mode 100644
index 8ce1d1b..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests covering rename.
- */
-public class ITestOzoneContractRename extends AbstractContractRenameTest {
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HDFS-13609. [SBN read] Edit Tail
Fast Path Part 3: NameNode-side changes to support tailing edits via RPC.
Contributed by Erik Krogen.
Posted by xk...@apache.org.
HDFS-13609. [SBN read] Edit Tail Fast Path Part 3: NameNode-side changes to support tailing edits via RPC. Contributed by Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83109733
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83109733
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83109733
Branch: refs/heads/HDFS-12943
Commit: 831097335833681505af257e15362bfef66987ad
Parents: da14307
Author: Erik Krogen <xk...@apache.org>
Authored: Tue May 22 16:45:26 2018 -0700
Committer: Erik Krogen <xk...@apache.org>
Committed: Tue Jun 26 11:01:07 2018 +0530
----------------------------------------------------------------------
.../hdfs/qjournal/client/AsyncLogger.java | 7 +
.../hdfs/qjournal/client/AsyncLoggerSet.java | 14 ++
.../hdfs/qjournal/client/IPCLoggerChannel.java | 14 ++
.../qjournal/client/QuorumJournalManager.java | 111 +++++++++++++++-
.../server/namenode/EditLogFileInputStream.java | 44 +++++++
.../hdfs/server/namenode/ha/EditLogTailer.java | 6 +-
.../src/main/resources/hdfs-default.xml | 4 +-
.../client/TestQuorumJournalManager.java | 130 +++++++++++++++++++
.../client/TestQuorumJournalManagerUnit.java | 101 +++++++++++++-
.../namenode/TestEditLogFileInputStream.java | 18 +++
10 files changed, 439 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83109733/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java
index d2b48cc..7230ebc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java
@@ -22,6 +22,7 @@ import java.net.URL;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocol;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournaledEditsResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalStateResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
@@ -106,6 +107,12 @@ interface AsyncLogger {
* Begin a new epoch on the target node.
*/
public ListenableFuture<NewEpochResponseProto> newEpoch(long epoch);
+
+ /**
+ * Fetch journaled edits from the cache.
+ */
+ public ListenableFuture<GetJournaledEditsResponseProto> getJournaledEdits(
+ long fromTxnId, int maxTransactions);
/**
* Fetch the list of edit logs available on the remote node.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83109733/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java
index d46c2cf..0366466 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java
@@ -25,6 +25,7 @@ import java.util.concurrent.TimeoutException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournaledEditsResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalStateResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
@@ -261,6 +262,19 @@ class AsyncLoggerSet {
return QuorumCall.create(calls);
}
+ public QuorumCall<AsyncLogger, GetJournaledEditsResponseProto>
+ getJournaledEdits(long fromTxnId, int maxTransactions) {
+ Map<AsyncLogger,
+ ListenableFuture<GetJournaledEditsResponseProto>> calls
+ = Maps.newHashMap();
+ for (AsyncLogger logger : loggers) {
+ ListenableFuture<GetJournaledEditsResponseProto> future =
+ logger.getJournaledEdits(fromTxnId, maxTransactions);
+ calls.put(logger, future);
+ }
+ return QuorumCall.create(calls);
+ }
+
public QuorumCall<AsyncLogger, RemoteEditLogManifest> getEditLogManifest(
long fromTxnId, boolean inProgressOk) {
Map<AsyncLogger,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83109733/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
index 3036735..94f72ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.qjournal.protocol.JournalOutOfSyncException;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocol;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetEditLogManifestResponseProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournaledEditsResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalStateResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
@@ -559,6 +560,19 @@ public class IPCLoggerChannel implements AsyncLogger {
}
@Override
+ public ListenableFuture<GetJournaledEditsResponseProto> getJournaledEdits(
+ long fromTxnId, int maxTransactions) {
+ return parallelExecutor.submit(
+ new Callable<GetJournaledEditsResponseProto>() {
+ @Override
+ public GetJournaledEditsResponseProto call() throws IOException {
+ return getProxy().getJournaledEdits(journalId, nameServiceId,
+ fromTxnId, maxTransactions);
+ }
+ });
+ }
+
+ @Override
public ListenableFuture<RemoteEditLogManifest> getEditLogManifest(
final long fromTxnId, final boolean inProgressOk) {
return parallelExecutor.submit(new Callable<RemoteEditLogManifest>() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83109733/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index 7a70a3d..a367b65 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -36,6 +37,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournaledEditsResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalStateResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
@@ -67,6 +69,14 @@ import com.google.protobuf.TextFormat;
public class QuorumJournalManager implements JournalManager {
static final Log LOG = LogFactory.getLog(QuorumJournalManager.class);
+ // This config is not publicly exposed
+ static final String QJM_RPC_MAX_TXNS_KEY =
+ "dfs.ha.tail-edits.qjm.rpc.max-txns";
+ static final int QJM_RPC_MAX_TXNS_DEFAULT = 5000;
+
+ // Maximum number of transactions to fetch at a time when using the
+ // RPC edit fetch mechanism
+ private final int maxTxnsPerRpc;
// Timeouts for which the QJM will wait for each of the following actions.
private final int startSegmentTimeoutMs;
private final int prepareRecoveryTimeoutMs;
@@ -127,6 +137,10 @@ public class QuorumJournalManager implements JournalManager {
this.connectionFactory = URLConnectionFactory
.newDefaultURLConnectionFactory(conf);
+ this.maxTxnsPerRpc =
+ conf.getInt(QJM_RPC_MAX_TXNS_KEY, QJM_RPC_MAX_TXNS_DEFAULT);
+ Preconditions.checkArgument(maxTxnsPerRpc > 0,
+ "Must specify %s greater than 0!", QJM_RPC_MAX_TXNS_KEY);
// Configure timeouts.
this.startSegmentTimeoutMs = conf.getInt(
DFSConfigKeys.DFS_QJOURNAL_START_SEGMENT_TIMEOUT_KEY,
@@ -471,17 +485,104 @@ public class QuorumJournalManager implements JournalManager {
public void selectInputStreams(Collection<EditLogInputStream> streams,
long fromTxnId, boolean inProgressOk,
boolean onlyDurableTxns) throws IOException {
+ if (inProgressOk) {
+ LOG.info("Tailing edits starting from txn ID " + fromTxnId +
+ " via RPC mechanism");
+ try {
+ Collection<EditLogInputStream> rpcStreams = new ArrayList<>();
+ selectRpcInputStreams(rpcStreams, fromTxnId, onlyDurableTxns);
+ streams.addAll(rpcStreams);
+ return;
+ } catch (IOException ioe) {
+ LOG.warn("Encountered exception while tailing edits >= " + fromTxnId +
+ " via RPC; falling back to streaming.", ioe);
+ }
+ }
+ selectStreamingInputStreams(streams, fromTxnId, inProgressOk,
+ onlyDurableTxns);
+ }
+
+ /**
+ * Select input streams from the journals, specifically using the RPC
+ * mechanism optimized for low latency.
+ *
+ * @param streams The collection to store the return streams into.
+ * @param fromTxnId Select edits starting from this transaction ID
+ * @param onlyDurableTxns Iff true, only include transactions which have been
+ * committed to a quorum of the journals.
+ * @throws IOException Upon issues, including cache misses on the journals.
+ */
+ private void selectRpcInputStreams(Collection<EditLogInputStream> streams,
+ long fromTxnId, boolean onlyDurableTxns) throws IOException {
+ QuorumCall<AsyncLogger, GetJournaledEditsResponseProto> q =
+ loggers.getJournaledEdits(fromTxnId, maxTxnsPerRpc);
+ Map<AsyncLogger, GetJournaledEditsResponseProto> responseMap =
+ loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs,
+ "selectRpcInputStreams");
+ assert responseMap.size() >= loggers.getMajoritySize() :
+ "Quorum call returned without a majority";
+ List<Integer> responseCounts = new ArrayList<>();
+ for (GetJournaledEditsResponseProto resp : responseMap.values()) {
+ responseCounts.add(resp.getTxnCount());
+ }
+ Collections.sort(responseCounts);
+ int highestTxnCount = responseCounts.get(responseCounts.size() - 1);
+ if (LOG.isDebugEnabled() || highestTxnCount < 0) {
+ StringBuilder msg = new StringBuilder("Requested edits starting from ");
+ msg.append(fromTxnId).append("; got ").append(responseMap.size())
+ .append(" responses: <");
+ for (Map.Entry<AsyncLogger, GetJournaledEditsResponseProto> ent :
+ responseMap.entrySet()) {
+ msg.append("[").append(ent.getKey()).append(", ")
+ .append(ent.getValue().getTxnCount()).append("],");
+ }
+ msg.append(">");
+ if (highestTxnCount < 0) {
+ throw new IOException("Did not get any valid JournaledEdits " +
+ "responses: " + msg);
+ } else {
+ LOG.debug(msg.toString());
+ }
+ }
+
+ int maxAllowedTxns = !onlyDurableTxns ? highestTxnCount :
+ responseCounts.get(responseCounts.size() - loggers.getMajoritySize());
+ if (maxAllowedTxns == 0) {
+ LOG.debug("No new edits available in logs; requested starting from " +
+ "ID " + fromTxnId);
+ return;
+ }
+ LOG.info("Selected loggers with >= " + maxAllowedTxns +
+ " transactions starting from " + fromTxnId);
+ PriorityQueue<EditLogInputStream> allStreams = new PriorityQueue<>(
+ JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR);
+ for (GetJournaledEditsResponseProto resp : responseMap.values()) {
+ long endTxnId = fromTxnId - 1 +
+ Math.min(maxAllowedTxns, resp.getTxnCount());
+ allStreams.add(EditLogFileInputStream.fromByteString(
+ resp.getEditLog(), fromTxnId, endTxnId, true));
+ }
+ JournalSet.chainAndMakeRedundantStreams(streams, allStreams, fromTxnId);
+ }
+
+ /**
+ * Select input streams from the journals, specifically using the streaming
+ * mechanism optimized for resiliency / bulk load.
+ */
+ private void selectStreamingInputStreams(
+ Collection<EditLogInputStream> streams, long fromTxnId,
+ boolean inProgressOk, boolean onlyDurableTxns) throws IOException {
QuorumCall<AsyncLogger, RemoteEditLogManifest> q =
loggers.getEditLogManifest(fromTxnId, inProgressOk);
Map<AsyncLogger, RemoteEditLogManifest> resps =
loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs,
- "selectInputStreams");
-
- LOG.debug("selectInputStream manifests:\n" +
+ "selectStreamingInputStreams");
+
+ LOG.debug("selectStreamingInputStream manifests:\n" +
Joiner.on("\n").withKeyValueSeparator(": ").join(resps));
-
- final PriorityQueue<EditLogInputStream> allStreams =
+
+ final PriorityQueue<EditLogInputStream> allStreams =
new PriorityQueue<EditLogInputStream>(64,
JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR);
for (Map.Entry<AsyncLogger, RemoteEditLogManifest> e : resps.entrySet()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83109733/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
index 36c2232..4e9b174 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode;
+import com.google.protobuf.ByteString;
import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.EOFException;
@@ -119,6 +120,23 @@ public class EditLogFileInputStream extends EditLogInputStream {
return new EditLogFileInputStream(new URLLog(connectionFactory, url),
startTxId, endTxId, inProgress);
}
+
+ /**
+ * Create an EditLogInputStream from a {@link ByteString}, i.e. an in-memory
+ * collection of bytes.
+ *
+ * @param bytes The byte string to read from
+ * @param startTxId the expected starting transaction ID
+ * @param endTxId the expected ending transaction ID
+ * @param inProgress whether the log is in-progress
+ * @return An edit stream to read from
+ */
+ public static EditLogInputStream fromByteString(ByteString bytes,
+ long startTxId, long endTxId, boolean inProgress) {
+ return new EditLogFileInputStream(new ByteStringLog(bytes,
+ String.format("ByteStringEditLog[%d, %d]", startTxId, endTxId)),
+ startTxId, endTxId, inProgress);
+ }
private EditLogFileInputStream(LogSource log,
long firstTxId, long lastTxId,
@@ -376,6 +394,32 @@ public class EditLogFileInputStream extends EditLogInputStream {
public long length();
public String getName();
}
+
+ private static class ByteStringLog implements LogSource {
+ private final ByteString bytes;
+ private final String name;
+
+ public ByteStringLog(ByteString bytes, String name) {
+ this.bytes = bytes;
+ this.name = name;
+ }
+
+ @Override
+ public InputStream getInputStream() {
+ return bytes.newInput();
+ }
+
+ @Override
+ public long length() {
+ return bytes.size();
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ }
private static class FileLog implements LogSource {
private final File file;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83109733/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
index 2003f94..7215b65 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
@@ -147,7 +147,11 @@ public class EditLogTailer {
private int maxRetries;
/**
- * Whether the tailer should tail the in-progress edit log segments.
+ * Whether the tailer should tail the in-progress edit log segments. If true,
+ * this will also attempt to optimize for latency when tailing the edit logs
+ * (if using the
+ * {@link org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager}, this
+ * implies using the RPC-based mechanism to tail edits).
*/
private final boolean inProgressOk;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83109733/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 9b109a7..a0fd1bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3124,7 +3124,9 @@
<description>
Whether enable standby namenode to tail in-progress edit logs.
Clients might want to turn it on when they want Standby NN to have
- more up-to-date data.
+ more up-to-date data. When using the QuorumJournalManager, this enables
+ tailing of edit logs via the RPC-based mechanism, rather than streaming,
+ which allows for much fresher data.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83109733/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
index 69856ae..cac8ba5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
@@ -44,6 +44,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster;
import org.apache.hadoop.hdfs.qjournal.QJMTestUtil;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
@@ -91,6 +92,10 @@ public class TestQuorumJournalManager {
conf = new Configuration();
// Don't retry connections - it just slows down the tests.
conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
+ // Turn off IPC client caching to handle daemon restarts.
+ conf.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
+ conf.setBoolean(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, true);
cluster = new MiniJournalCluster.Builder(conf)
.baseDir(GenericTestUtils.getRandomizedTestDir().getAbsolutePath())
@@ -959,6 +964,131 @@ public class TestQuorumJournalManager {
qjm2.selectInputStreams(streams, 1, true, true);
verifyEdits(streams, 1, 8);
}
+
+ @Test
+ public void testSelectViaRpcWithDurableTransactions() throws Exception {
+ // Two loggers will have up to ID 5, one will have up to ID 6
+ failLoggerAtTxn(spies.get(0), 6);
+ failLoggerAtTxn(spies.get(1), 6);
+ EditLogOutputStream stm =
+ qjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ writeTxns(stm, 1, 5);
+ try {
+ writeTxns(stm, 6, 1);
+ fail("Did not fail to write when only a minority succeeded");
+ } catch (QuorumException qe) {
+ GenericTestUtils.assertExceptionContains(
+ "too many exceptions to achieve quorum size 2/3", qe);
+ }
+
+ List<EditLogInputStream> streams = new ArrayList<>();
+ qjm.selectInputStreams(streams, 1, true, true);
+ verifyEdits(streams, 1, 5);
+ IOUtils.closeStreams(streams.toArray(new Closeable[0]));
+ for (AsyncLogger logger : spies) {
+ Mockito.verify(logger, Mockito.times(1)).getJournaledEdits(1,
+ QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+ }
+ }
+
+ @Test
+ public void testSelectViaRpcWithoutDurableTransactions() throws Exception {
+ setupLoggers345();
+ futureThrows(new IOException()).when(spies.get(1)).getJournaledEdits(1,
+ QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+
+ List<EditLogInputStream> streams = new ArrayList<>();
+ qjm.selectInputStreams(streams, 1, true, false);
+ verifyEdits(streams, 1, 5);
+ IOUtils.closeStreams(streams.toArray(new Closeable[0]));
+ for (AsyncLogger logger : spies) {
+ Mockito.verify(logger, Mockito.times(1)).getJournaledEdits(1,
+ QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+ }
+ }
+
+ @Test
+ public void testSelectViaRpcOneDeadJN() throws Exception {
+ EditLogOutputStream stm =
+ qjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ writeTxns(stm, 1, 10);
+
+ cluster.getJournalNode(0).stopAndJoin(0);
+
+ List<EditLogInputStream> streams = new ArrayList<>();
+ qjm.selectInputStreams(streams, 1, true, false);
+ verifyEdits(streams, 1, 10);
+ IOUtils.closeStreams(streams.toArray(new Closeable[0]));
+ }
+
+ @Test
+ public void testSelectViaRpcTwoDeadJNs() throws Exception {
+ EditLogOutputStream stm =
+ qjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ writeTxns(stm, 1, 10);
+
+ cluster.getJournalNode(0).stopAndJoin(0);
+ cluster.getJournalNode(1).stopAndJoin(0);
+
+ try {
+ qjm.selectInputStreams(new ArrayList<>(), 1, true, false);
+ fail("");
+ } catch (QuorumException qe) {
+ GenericTestUtils.assertExceptionContains(
+ "too many exceptions to achieve quorum size 2/3", qe);
+ }
+ }
+
+ @Test
+ public void testSelectViaRpcTwoJNsError() throws Exception {
+ EditLogOutputStream stm =
+ qjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ writeTxns(stm, 1, 10);
+ writeTxns(stm, 11, 1);
+
+ futureThrows(new IOException()).when(spies.get(0)).getJournaledEdits(1,
+ QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+ futureThrows(new IOException()).when(spies.get(1)).getJournaledEdits(1,
+ QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+
+ List<EditLogInputStream> streams = new ArrayList<>();
+ qjm.selectInputStreams(streams, 1, true, true);
+ // This should still succeed as the QJM should fall back to the streaming
+ // mechanism for fetching edits
+ verifyEdits(streams, 1, 11);
+ IOUtils.closeStreams(streams.toArray(new Closeable[0]));
+
+ for (AsyncLogger logger : spies) {
+ Mockito.verify(logger, Mockito.times(1)).getEditLogManifest(1, true);
+ }
+ }
+
+ @Test
+ public void testSelectViaRpcAfterJNRestart() throws Exception {
+ EditLogOutputStream stm =
+ qjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ writeTxns(stm, 1, 10);
+ qjm.finalizeLogSegment(1, 10);
+
+ // Close to avoid connections hanging around after the JNs are restarted
+ for (int i = 0; i < cluster.getNumNodes(); i++) {
+ cluster.restartJournalNode(i);
+ }
+ cluster.waitActive();
+
+ qjm = createSpyingQJM();
+ spies = qjm.getLoggerSetForTests().getLoggersForTests();
+ List<EditLogInputStream> streams = new ArrayList<>();
+ qjm.selectInputStreams(streams, 1, true, true);
+ // This should still succeed as the QJM should fall back to the streaming
+ // mechanism for fetching edits
+ verifyEdits(streams, 1, 10);
+ IOUtils.closeStreams(streams.toArray(new Closeable[0]));
+
+ for (AsyncLogger logger : spies) {
+ Mockito.verify(logger, Mockito.times(1)).getEditLogManifest(1, true);
+ }
+ }
private QuorumJournalManager createSpyingQJM()
throws IOException, URISyntaxException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83109733/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java
index 75dcf2f..0158eb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java
@@ -17,10 +17,13 @@
*/
package org.apache.hadoop.hdfs.qjournal.client;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.eq;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.net.URI;
import java.util.List;
@@ -28,11 +31,11 @@ import java.util.List;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.qjournal.client.AsyncLogger;
-import org.apache.hadoop.hdfs.qjournal.client.QuorumException;
-import org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournaledEditsResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournalStateResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
+import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
+import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream;
import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream;
import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
@@ -44,11 +47,15 @@ import org.mockito.Mockito;
import org.mockito.stubbing.Stubber;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
+import com.google.protobuf.ByteString;
import static org.apache.hadoop.hdfs.qjournal.QJMTestUtil.writeOp;
+import static org.apache.hadoop.hdfs.qjournal.QJMTestUtil.createTxnData;
+import static org.apache.hadoop.hdfs.qjournal.QJMTestUtil.verifyEdits;
/**
* True unit tests for QuorumJournalManager
@@ -215,6 +222,94 @@ public class TestQuorumJournalManagerUnit {
Mockito.verify(spyLoggers.get(0)).setCommittedTxId(1L);
}
+ @Test
+ public void testReadRpcInputStreams() throws Exception {
+ for (int jn = 0; jn < 3; jn++) {
+ futureReturns(getJournaledEditsReponse(1, 3))
+ .when(spyLoggers.get(jn)).getJournaledEdits(1,
+ QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+ }
+
+ List<EditLogInputStream> streams = Lists.newArrayList();
+ qjm.selectInputStreams(streams, 1, true, true);
+ assertEquals(1, streams.size());
+ verifyEdits(streams, 1, 3);
+ }
+
+ @Test
+ public void testReadRpcMismatchedInputStreams() throws Exception {
+ for (int jn = 0; jn < 3; jn++) {
+ futureReturns(getJournaledEditsReponse(1, jn + 1))
+ .when(spyLoggers.get(jn)).getJournaledEdits(1,
+ QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+ }
+
+ List<EditLogInputStream> streams = Lists.newArrayList();
+ qjm.selectInputStreams(streams, 1, true, true);
+ assertEquals(1, streams.size());
+ verifyEdits(streams, 1, 2);
+ }
+
+ @Test
+ public void testReadRpcInputStreamsOneSlow() throws Exception {
+ for (int jn = 0; jn < 2; jn++) {
+ futureReturns(getJournaledEditsReponse(1, jn + 1))
+ .when(spyLoggers.get(jn)).getJournaledEdits(1,
+ QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+ }
+ Mockito.doReturn(SettableFuture.create())
+ .when(spyLoggers.get(2)).getJournaledEdits(1,
+ QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+
+ List<EditLogInputStream> streams = Lists.newArrayList();
+ qjm.selectInputStreams(streams, 1, true, true);
+ assertEquals(1, streams.size());
+ verifyEdits(streams, 1, 1);
+ }
+
+ @Test
+ public void testReadRpcInputStreamsOneException() throws Exception {
+ for (int jn = 0; jn < 2; jn++) {
+ futureReturns(getJournaledEditsReponse(1, jn + 1))
+ .when(spyLoggers.get(jn)).getJournaledEdits(1,
+ QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+ }
+ futureThrows(new IOException()).when(spyLoggers.get(2))
+ .getJournaledEdits(1, QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+
+ List<EditLogInputStream> streams = Lists.newArrayList();
+ qjm.selectInputStreams(streams, 1, true, true);
+ assertEquals(1, streams.size());
+ verifyEdits(streams, 1, 1);
+ }
+
+ @Test
+ public void testReadRpcInputStreamsNoNewEdits() throws Exception {
+ for (int jn = 0; jn < 3; jn++) {
+ futureReturns(GetJournaledEditsResponseProto.newBuilder()
+ .setTxnCount(0).setEditLog(ByteString.EMPTY).build())
+ .when(spyLoggers.get(jn))
+ .getJournaledEdits(1, QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+ }
+
+ List<EditLogInputStream> streams = Lists.newArrayList();
+ qjm.selectInputStreams(streams, 1, true, true);
+ assertEquals(0, streams.size());
+ }
+
+ private GetJournaledEditsResponseProto getJournaledEditsReponse(
+ int startTxn, int numTxns) throws Exception {
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ EditLogFileOutputStream.writeHeader(
+ NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION,
+ new DataOutputStream(byteStream));
+ byteStream.write(createTxnData(startTxn, numTxns));
+ return GetJournaledEditsResponseProto.newBuilder()
+ .setTxnCount(numTxns)
+ .setEditLog(ByteString.copyFrom(byteStream.toByteArray()))
+ .build();
+ }
+
private EditLogOutputStream createLogSegment() throws IOException {
futureReturns(null).when(spyLoggers.get(0)).startLogSegment(Mockito.anyLong(),
Mockito.eq(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83109733/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileInputStream.java
index cd329a6..0843bfe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileInputStream.java
@@ -32,6 +32,7 @@ import java.net.HttpURLConnection;
import java.net.URL;
import java.util.EnumMap;
+import com.google.protobuf.ByteString;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -80,6 +81,23 @@ public class TestEditLogFileInputStream {
elis.close();
}
+ @Test
+ public void testByteStringLog() throws Exception {
+ ByteString bytes = ByteString.copyFrom(FAKE_LOG_DATA);
+ EditLogInputStream elis = EditLogFileInputStream.fromByteString(bytes,
+ HdfsServerConstants.INVALID_TXID, HdfsServerConstants.INVALID_TXID,
+ true);
+ // Read the edit log and verify that all of the data is present
+ EnumMap<FSEditLogOpCodes, Holder<Integer>> counts = FSImageTestUtil
+ .countEditLogOpTypes(elis);
+ assertThat(counts.get(FSEditLogOpCodes.OP_ADD).held, is(1));
+ assertThat(counts.get(FSEditLogOpCodes.OP_SET_GENSTAMP_V1).held, is(1));
+ assertThat(counts.get(FSEditLogOpCodes.OP_CLOSE).held, is(1));
+
+ assertEquals(FAKE_LOG_DATA.length, elis.length());
+ elis.close();
+ }
+
/**
* Regression test for HDFS-8965 which verifies that
* FSEditLogFileInputStream#scanOp verifies Op checksums.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HADOOP-15523. Shell command
timeout given is in seconds whereas it is taken as millisec while scheduling.
Contributed by Bilwa S T.
Posted by xk...@apache.org.
HADOOP-15523. Shell command timeout given is in seconds whereas it is taken as millisec while scheduling. Contributed by Bilwa S T.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3905fdb7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3905fdb7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3905fdb7
Branch: refs/heads/HDFS-12943
Commit: 3905fdb793e6370243d05d0c3036ca69898fe3fb
Parents: 8762e9c
Author: Surendra Singh Lilhore <su...@apache.org>
Authored: Sun Jun 17 12:12:01 2018 +0530
Committer: Surendra Singh Lilhore <su...@apache.org>
Committed: Sun Jun 17 12:12:01 2018 +0530
----------------------------------------------------------------------
.../fs/CommonConfigurationKeysPublic.java | 4 +-
.../security/ShellBasedUnixGroupsMapping.java | 10 ++---
.../main/java/org/apache/hadoop/util/Shell.java | 2 +-
.../TestShellBasedUnixGroupsMapping.java | 39 ++++++++++++++++++--
4 files changed, 43 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3905fdb7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 9e0ba20..c7f32f9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -542,7 +542,7 @@ public class CommonConfigurationKeysPublic {
* <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
* core-default.xml</a>
*/
- public static final String HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS =
+ public static final String HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY =
"hadoop.security.groups.shell.command.timeout";
/**
* @see
@@ -550,7 +550,7 @@ public class CommonConfigurationKeysPublic {
* core-default.xml</a>
*/
public static final long
- HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT =
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT =
0L;
/**
* @see
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3905fdb7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
index 94698d8..976ddba 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.security;
import java.io.IOException;
-import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.StringTokenizer;
@@ -52,7 +51,8 @@ public class ShellBasedUnixGroupsMapping extends Configured
protected static final Logger LOG =
LoggerFactory.getLogger(ShellBasedUnixGroupsMapping.class);
- private long timeout = 0L;
+ private long timeout = CommonConfigurationKeys.
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT;
private static final List<String> EMPTY_GROUPS = new LinkedList<>();
@Override
@@ -61,10 +61,10 @@ public class ShellBasedUnixGroupsMapping extends Configured
if (conf != null) {
timeout = conf.getTimeDuration(
CommonConfigurationKeys.
- HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS,
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY,
CommonConfigurationKeys.
- HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT,
- TimeUnit.SECONDS);
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3905fdb7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index 0b76f0d..46a0fcc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -1191,7 +1191,7 @@ public abstract class Shell {
/**
* Returns the timeout value set for the executor's sub-commands.
- * @return The timeout value in seconds
+ * @return The timeout value in milliseconds
*/
@VisibleForTesting
public long getTimeoutInterval() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3905fdb7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java
index d3c9538..8c1339d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java
@@ -173,6 +173,37 @@ public class TestShellBasedUnixGroupsMapping {
assertTrue(groups.contains("zzz"));
}
+ public long getTimeoutInterval(String timeout) {
+ Configuration conf = new Configuration();
+ String userName = "foobarnonexistinguser";
+ conf.set(
+ CommonConfigurationKeys.HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY,
+ timeout);
+ TestDelayedGroupCommand mapping = ReflectionUtils
+ .newInstance(TestDelayedGroupCommand.class, conf);
+ ShellCommandExecutor executor = mapping.createGroupExecutor(userName);
+ return executor.getTimeoutInterval();
+ }
+
+ @Test
+ public void testShellTimeOutConf() {
+
+ // Test a 1 second max-runtime timeout
+ assertEquals(
+ "Expected the group names executor to carry the configured timeout",
+ 1000L, getTimeoutInterval("1s"));
+
+ // Test a 1 minute max-runtime timeout
+ assertEquals(
+ "Expected the group names executor to carry the configured timeout",
+ 60000L, getTimeoutInterval("1m"));
+
+ // Test a 1 millisecond max-runtime timeout
+ assertEquals(
+ "Expected the group names executor to carry the configured timeout",
+ 1L, getTimeoutInterval("1"));
+ }
+
private class TestGroupResolvable
extends ShellBasedUnixGroupsMapping {
/**
@@ -222,7 +253,7 @@ public class TestShellBasedUnixGroupsMapping {
private static class TestDelayedGroupCommand
extends ShellBasedUnixGroupsMapping {
- private Long timeoutSecs = 2L;
+ private Long timeoutSecs = 1L;
TestDelayedGroupCommand() {
super();
@@ -249,12 +280,12 @@ public class TestShellBasedUnixGroupsMapping {
String userName = "foobarnonexistinguser";
String commandTimeoutMessage =
"ran longer than the configured timeout limit";
- long testTimeout = 1L;
+ long testTimeout = 500L;
// Test a 1 second max-runtime timeout
conf.setLong(
CommonConfigurationKeys.
- HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS,
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY,
testTimeout);
TestDelayedGroupCommand mapping =
@@ -306,7 +337,7 @@ public class TestShellBasedUnixGroupsMapping {
conf = new Configuration();
long defaultTimeout =
CommonConfigurationKeys.
- HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT;
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT;
mapping =
ReflectionUtils.newInstance(TestDelayedGroupCommand.class, conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HADOOP-14918. Remove the Local
Dynamo DB test option. Contributed by Gabor Bota.
Posted by xk...@apache.org.
HADOOP-14918. Remove the Local Dynamo DB test option. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b089a067
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b089a067
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b089a067
Branch: refs/heads/HDFS-12943
Commit: b089a06793d94d42b7da1b7566e366ceb748e081
Parents: d6ee429
Author: Sean Mackrory <ma...@apache.org>
Authored: Wed Jun 20 16:10:36 2018 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Wed Jun 20 16:45:08 2018 -0600
----------------------------------------------------------------------
hadoop-project/pom.xml | 5 -
hadoop-tools/hadoop-aws/pom.xml | 42 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 11 +
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 1 +
.../hadoop/fs/s3a/AbstractS3ATestBase.java | 21 +-
.../apache/hadoop/fs/s3a/S3ATestConstants.java | 1 -
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 34 +-
.../fs/s3a/commit/staging/StagingTestBase.java | 30 +-
.../s3a/s3guard/DynamoDBLocalClientFactory.java | 160 -----
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 4 +-
.../s3a/s3guard/TestDynamoDBMetadataStore.java | 589 -------------------
11 files changed, 73 insertions(+), 825 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 8cb5bfc..ed0187b 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -2060,10 +2060,5 @@
</profiles>
<repositories>
- <repository>
- <id>dynamodb-local-oregon</id>
- <name>DynamoDB Local Release Repository</name>
- <url>https://s3-us-west-2.amazonaws.com/dynamodb-local/release</url>
- </repository>
</repositories>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 24ed11d..c6dddb0 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -36,7 +36,6 @@
<downloadSources>true</downloadSources>
<hadoop.tmp.dir>${project.build.directory}/test</hadoop.tmp.dir>
- <dynamodb.local.version>1.11.86</dynamodb.local.version>
<!-- are scale tests enabled ? -->
<fs.s3a.scale.test.enabled>unset</fs.s3a.scale.test.enabled>
<!-- Size in MB of huge files. -->
@@ -49,6 +48,8 @@
<fs.s3a.s3guard.test.enabled>false</fs.s3a.s3guard.test.enabled>
<fs.s3a.s3guard.test.authoritative>false</fs.s3a.s3guard.test.authoritative>
<fs.s3a.s3guard.test.implementation>local</fs.s3a.s3guard.test.implementation>
+ <!-- Set a longer timeout for integration test (in milliseconds) -->
+ <test.integration.timeout>200000</test.integration.timeout>
</properties>
@@ -162,6 +163,7 @@
<fs.s3a.s3guard.test.authoritative>${fs.s3a.s3guard.test.authoritative}</fs.s3a.s3guard.test.authoritative>
<fs.s3a.s3guard.test.implementation>${fs.s3a.s3guard.test.implementation}</fs.s3a.s3guard.test.implementation>
+ <test.default.timeout>${test.integration.timeout}</test.default.timeout>
</systemPropertyVariables>
<!-- Some tests cannot run in parallel. Tests that cover -->
<!-- access to the root directory must run in isolation -->
@@ -299,23 +301,10 @@
</properties>
</profile>
- <!-- Switch to DynamoDBLocal for S3Guard. Has no effect unless S3Guard is enabled -->
- <profile>
- <id>dynamodblocal</id>
- <activation>
- <property>
- <name>dynamodblocal</name>
- </property>
- </activation>
- <properties>
- <fs.s3a.s3guard.test.implementation>dynamodblocal</fs.s3a.s3guard.test.implementation>
- </properties>
- </profile>
-
<!-- Switch S3Guard from Authoritative=false to true
Has no effect unless S3Guard is enabled -->
<profile>
- <id>non-auth</id>
+ <id>auth</id>
<activation>
<property>
<name>auth</name>
@@ -346,6 +335,9 @@
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
+ <systemPropertyVariables>
+ <test.default.timeout>${test.integration.timeout}</test.default.timeout>
+ </systemPropertyVariables>
</configuration>
</plugin>
<plugin>
@@ -418,26 +410,6 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>DynamoDBLocal</artifactId>
- <version>${dynamodb.local.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-http</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 4c95843..c521936 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -402,6 +402,17 @@ public final class Constants {
"fs.s3a.s3guard.ddb.table";
/**
+ * Test table name to use during DynamoDB integration test.
+ *
+ * The table will be modified, and deleted in the end of the tests.
+ * If this value is not set, the integration tests that would be destructive
+ * won't run.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_TEST_TABLE_NAME_KEY =
+ "fs.s3a.s3guard.ddb.test.table";
+
+ /**
* Whether to create the DynamoDB table if the table does not exist.
*/
@InterfaceStability.Unstable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index b942ed7..0ab8696 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -261,6 +261,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
@Override
@Retries.OnceRaw
public void initialize(FileSystem fs) throws IOException {
+ Preconditions.checkNotNull(fs, "Null filesystem");
Preconditions.checkArgument(fs instanceof S3AFileSystem,
"DynamoDBMetadataStore only supports S3A filesystem.");
owner = (S3AFileSystem) fs;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
index 73e71f4..f22af49 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
@@ -29,13 +29,10 @@ import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
import java.io.IOException;
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
-import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_ENABLED;
/**
* An extension of the contract test base set up for S3A tests.
@@ -78,23 +75,7 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase
*/
@Override
protected Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- // patch in S3Guard options
- maybeEnableS3Guard(conf);
- // set hadoop temp dir to a default value
- String testUniqueForkId =
- System.getProperty(TEST_UNIQUE_FORK_ID);
- String tmpDir = conf.get(Constants.HADOOP_TMP_DIR, "target/build/test");
- if (testUniqueForkId != null) {
- // patch temp dir for the specific branch
- tmpDir = tmpDir + File.pathSeparatorChar + testUniqueForkId;
- conf.set(Constants.HADOOP_TMP_DIR, tmpDir);
- }
- conf.set(Constants.BUFFER_DIR, tmpDir);
- // add this so that even on tests where the FS is shared,
- // the FS is always "magic"
- conf.setBoolean(MAGIC_COMMITTER_ENABLED, true);
- return conf;
+ return S3ATestUtils.prepareTestConfiguration(super.createConfiguration());
}
protected Configuration getConfiguration() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
index 7f7802d..0f7b418 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
@@ -143,7 +143,6 @@ public interface S3ATestConstants {
String TEST_S3GUARD_IMPLEMENTATION = TEST_S3GUARD_PREFIX + ".implementation";
String TEST_S3GUARD_IMPLEMENTATION_LOCAL = "local";
String TEST_S3GUARD_IMPLEMENTATION_DYNAMO = "dynamo";
- String TEST_S3GUARD_IMPLEMENTATION_DYNAMODBLOCAL = "dynamodblocal";
String TEST_S3GUARD_IMPLEMENTATION_NONE = "none";
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 4414746..d259bf1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -30,9 +30,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.s3a.commit.CommitConstants;
-import org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory;
-import org.apache.hadoop.fs.s3a.s3guard.DynamoDBLocalClientFactory;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
import org.hamcrest.core.Is;
import org.junit.Assert;
@@ -42,6 +39,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
+import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -56,6 +54,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.S3AUtils.propagateBucketOptions;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_ENABLED;
import static org.junit.Assert.*;
/**
@@ -393,9 +392,6 @@ public final class S3ATestUtils {
case TEST_S3GUARD_IMPLEMENTATION_LOCAL:
implClass = S3GUARD_METASTORE_LOCAL;
break;
- case TEST_S3GUARD_IMPLEMENTATION_DYNAMODBLOCAL:
- conf.setClass(S3Guard.S3GUARD_DDB_CLIENT_FACTORY_IMPL,
- DynamoDBLocalClientFactory.class, DynamoDBClientFactory.class);
case TEST_S3GUARD_IMPLEMENTATION_DYNAMO:
implClass = S3GUARD_METASTORE_DYNAMO;
break;
@@ -490,6 +486,32 @@ public final class S3ATestUtils {
}
/**
+ * Patch a configuration for testing.
+ * This includes possibly enabling s3guard, setting up the local
+ * FS temp dir and anything else needed for test runs.
+ * @param conf configuration to patch
+ * @return the now-patched configuration
+ */
+ public static Configuration prepareTestConfiguration(final Configuration conf) {
+ // patch in S3Guard options
+ maybeEnableS3Guard(conf);
+ // set hadoop temp dir to a default value
+ String testUniqueForkId =
+ System.getProperty(TEST_UNIQUE_FORK_ID);
+ String tmpDir = conf.get(HADOOP_TMP_DIR, "target/build/test");
+ if (testUniqueForkId != null) {
+ // patch temp dir for the specific branch
+ tmpDir = tmpDir + File.pathSeparatorChar + testUniqueForkId;
+ conf.set(HADOOP_TMP_DIR, tmpDir);
+ }
+ conf.set(BUFFER_DIR, tmpDir);
+ // add this so that even on tests where the FS is shared,
+ // the FS is always "magic"
+ conf.setBoolean(MAGIC_COMMITTER_ENABLED, true);
+ return conf;
+ }
+
+ /**
* Helper class to do diffs of metrics.
*/
public static final class MetricDiff {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java
index 38d5156..d81c747 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java
@@ -49,6 +49,7 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.mockito.invocation.InvocationOnMock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -515,6 +516,21 @@ public class StagingTestBase {
}
/**
+ * InvocationOnMock.getArgumentAt comes and goes with Mockito versions; this
+ * helper method is designed to be resilient to change.
+ * @param invocation invocation to query
+ * @param index argument index
+ * @param clazz class of return type
+ * @param <T> type of return
+ * @return the argument of the invocation, cast to the given type.
+ */
+ @SuppressWarnings("unchecked")
+ private static<T> T getArgumentAt(InvocationOnMock invocation, int index,
+ Class<T> clazz) {
+ return (T)invocation.getArguments()[index];
+ }
+
+ /**
* Instantiate mock client with the results and errors requested.
* @param results results to accrue
* @param errors when (if any) to fail
@@ -539,7 +555,7 @@ public class StagingTestBase {
"Mock Fail on init " + results.requests.size());
}
String uploadId = UUID.randomUUID().toString();
- InitiateMultipartUploadRequest req = invocation.getArgumentAt(
+ InitiateMultipartUploadRequest req = getArgumentAt(invocation,
0, InitiateMultipartUploadRequest.class);
results.requests.put(uploadId, req);
results.activeUploads.put(uploadId, req.getKey());
@@ -561,7 +577,7 @@ public class StagingTestBase {
throw new AmazonClientException(
"Mock Fail on upload " + results.parts.size());
}
- UploadPartRequest req = invocation.getArgumentAt(
+ UploadPartRequest req = getArgumentAt(invocation,
0, UploadPartRequest.class);
results.parts.add(req);
String etag = UUID.randomUUID().toString();
@@ -588,7 +604,7 @@ public class StagingTestBase {
throw new AmazonClientException(
"Mock Fail on commit " + results.commits.size());
}
- CompleteMultipartUploadRequest req = invocation.getArgumentAt(
+ CompleteMultipartUploadRequest req = getArgumentAt(invocation,
0, CompleteMultipartUploadRequest.class);
results.commits.add(req);
results.activeUploads.remove(req.getUploadId());
@@ -608,7 +624,7 @@ public class StagingTestBase {
throw new AmazonClientException(
"Mock Fail on abort " + results.aborts.size());
}
- AbortMultipartUploadRequest req = invocation.getArgumentAt(
+ AbortMultipartUploadRequest req = getArgumentAt(invocation,
0, AbortMultipartUploadRequest.class);
String id = req.getUploadId();
String p = results.activeUploads.remove(id);
@@ -630,7 +646,7 @@ public class StagingTestBase {
doAnswer(invocation -> {
LOG.debug("deleteObject for {}", mockClient);
synchronized (lock) {
- results.deletes.add(invocation.getArgumentAt(
+ results.deletes.add(getArgumentAt(invocation,
0, DeleteObjectRequest.class));
return null;
}
@@ -643,8 +659,8 @@ public class StagingTestBase {
LOG.debug("deleteObject for {}", mockClient);
synchronized (lock) {
results.deletes.add(new DeleteObjectRequest(
- invocation.getArgumentAt(0, String.class),
- invocation.getArgumentAt(1, String.class)
+ getArgumentAt(invocation, 0, String.class),
+ getArgumentAt(invocation, 1, String.class)
));
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java
deleted file mode 100644
index 9894ac4..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.File;
-import java.io.IOException;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
-import com.amazonaws.services.dynamodbv2.local.main.ServerRunner;
-import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer;
-import org.apache.commons.lang3.StringUtils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory;
-import org.apache.hadoop.net.ServerSocketUtil;
-
-import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
-import static org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory.DefaultDynamoDBClientFactory.getRegion;
-
-/**
- * A DynamoDBClientFactory implementation that creates AmazonDynamoDB clients
- * against an in-memory DynamoDBLocal server instance.
- *
- * You won't be charged bills for issuing any DynamoDB requests. However, the
- * DynamoDBLocal is considered a simulator of the DynamoDB web service, so it
- * may be stale or different. For example, the throttling is not yet supported
- * in DynamoDBLocal. This is for testing purpose only.
- *
- * To use this for creating DynamoDB client in tests:
- * <ol>
- * <li>
- * As all DynamoDBClientFactory implementations, this should be configured.
- * </li>
- * <li>
- * The singleton DynamoDBLocal server instance is started automatically when
- * creating the AmazonDynamoDB client for the first time. It still merits to
- * launch the server before all the tests and fail fast if error happens.
- * </li>
- * <li>
- * The server can be stopped explicitly, which is not actually needed in
- * tests as JVM termination will do that.
- * </li>
- * </ol>
- *
- * @see DefaultDynamoDBClientFactory
- */
-public class DynamoDBLocalClientFactory extends Configured
- implements DynamoDBClientFactory {
-
- /** The DynamoDBLocal dynamoDBLocalServer instance for testing. */
- private static DynamoDBProxyServer dynamoDBLocalServer;
- private static String ddbEndpoint;
-
- private static final String SYSPROP_SQLITE_LIB = "sqlite4java.library.path";
-
- @Override
- public AmazonDynamoDB createDynamoDBClient(String defaultRegion)
- throws IOException {
- startSingletonServer();
-
- final Configuration conf = getConf();
- // use the default credential provider chain
- conf.unset(AWS_CREDENTIALS_PROVIDER);
- final AWSCredentialsProvider credentials =
- createAWSCredentialProviderSet(null, conf);
- final ClientConfiguration awsConf =
- DefaultS3ClientFactory.createAwsConf(conf);
- // fail fast in case of service errors
- awsConf.setMaxErrorRetry(3);
-
- final String region = getRegion(conf, defaultRegion);
- LOG.info("Creating DynamoDBLocal client using endpoint {} in region {}",
- ddbEndpoint, region);
-
- return AmazonDynamoDBClientBuilder.standard()
- .withCredentials(credentials)
- .withClientConfiguration(awsConf)
- .withEndpointConfiguration(
- new AwsClientBuilder.EndpointConfiguration(ddbEndpoint, region))
- .build();
- }
-
- /**
- * Start a singleton in-memory DynamoDBLocal server if not started yet.
- * @throws IOException if any error occurs
- */
- public synchronized static void startSingletonServer() throws IOException {
- if (dynamoDBLocalServer != null) {
- return;
- }
-
- // Set this property if it has not been set elsewhere
- if (StringUtils.isEmpty(System.getProperty(SYSPROP_SQLITE_LIB))) {
- String projectBuildDir = System.getProperty("project.build.directory");
- if (StringUtils.isEmpty(projectBuildDir)) {
- projectBuildDir = "target";
- }
- // sqlite4java lib should have been copied to $projectBuildDir/native-libs
- System.setProperty(SYSPROP_SQLITE_LIB,
- projectBuildDir + File.separator + "native-libs");
- LOG.info("Setting {} -> {}",
- SYSPROP_SQLITE_LIB, System.getProperty(SYSPROP_SQLITE_LIB));
- }
-
- try {
- // Start an in-memory local DynamoDB instance
- final String port = String.valueOf(ServerSocketUtil.getPort(0, 100));
- ddbEndpoint = "http://localhost:" + port;
- dynamoDBLocalServer = ServerRunner.createServerFromCommandLineArgs(
- new String[]{"-inMemory", "-port", port});
- dynamoDBLocalServer.start();
- LOG.info("DynamoDBLocal singleton server was started at {}", ddbEndpoint);
- } catch (Exception t) {
- String msg = "Error starting DynamoDBLocal server at " + ddbEndpoint
- + " " + t;
- LOG.error(msg, t);
- throw new IOException(msg, t);
- }
- }
-
- /**
- * Stop the in-memory DynamoDBLocal server if it is started.
- * @throws IOException if any error occurs
- */
- public synchronized static void stopSingletonServer() throws IOException {
- if (dynamoDBLocalServer != null) {
- LOG.info("Shutting down the in-memory DynamoDBLocal server");
- try {
- dynamoDBLocalServer.stop();
- } catch (Throwable t) {
- String msg = "Error stopping DynamoDBLocal server at " + ddbEndpoint;
- LOG.error(msg, t);
- throw new IOException(msg, t);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
index 806940b..56618cb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -28,7 +28,6 @@ import java.util.Map;
import com.google.common.collect.Sets;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
@@ -43,6 +42,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.apache.hadoop.fs.s3a.Tristate;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.test.HadoopTestBase;
/**
* Main test class for MetadataStore implementations.
@@ -51,7 +51,7 @@ import org.apache.hadoop.io.IOUtils;
* If your implementation may return missing results for recently set paths,
* override {@link MetadataStoreTestBase#allowMissing()}.
*/
-public abstract class MetadataStoreTestBase extends Assert {
+public abstract class MetadataStoreTestBase extends HadoopTestBase {
private static final Logger LOG =
LoggerFactory.getLogger(MetadataStoreTestBase.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
deleted file mode 100644
index 5763b83..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
+++ /dev/null
@@ -1,589 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.services.dynamodbv2.document.DynamoDB;
-import com.amazonaws.services.dynamodbv2.document.Item;
-import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
-import com.amazonaws.services.dynamodbv2.document.Table;
-import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputDescription;
-import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
-import com.amazonaws.services.dynamodbv2.model.TableDescription;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.hadoop.fs.s3a.Tristate;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.MockS3ClientFactory;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3ClientFactory;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.*;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*;
-import static org.apache.hadoop.test.LambdaTestUtils.*;
-
-/**
- * Test that {@link DynamoDBMetadataStore} implements {@link MetadataStore}.
- *
- * In this unit test, we use an in-memory DynamoDBLocal server instead of real
- * AWS DynamoDB. An {@link S3AFileSystem} object is created and shared for
- * initializing {@link DynamoDBMetadataStore} objects. There are no real S3
- * request issued as the underlying AWS S3Client is mocked. You won't be
- * charged bills for AWS S3 or DynamoDB when you run this test.
- *
- * According to the base class, every test case will have independent contract
- * to create a new {@link DynamoDBMetadataStore} instance and initializes it.
- * A table will be created for each test by the test contract, and will be
- * destroyed after the test case finishes.
- */
-public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestDynamoDBMetadataStore.class);
- private static final String BUCKET = "TestDynamoDBMetadataStore";
- private static final String S3URI =
- URI.create(FS_S3A + "://" + BUCKET + "/").toString();
- public static final PrimaryKey
- VERSION_MARKER_PRIMARY_KEY = createVersionMarkerPrimaryKey(
- DynamoDBMetadataStore.VERSION_MARKER);
-
- /** The DynamoDB instance that can issue requests directly to server. */
- private static DynamoDB dynamoDB;
-
- @Rule
- public final Timeout timeout = new Timeout(60 * 1000);
-
- /**
- * Start the in-memory DynamoDBLocal server and initializes s3 file system.
- */
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- DynamoDBLocalClientFactory.startSingletonServer();
- try {
- dynamoDB = new DynamoDBMSContract().getMetadataStore().getDynamoDB();
- } catch (AmazonServiceException e) {
- final String msg = "Cannot initialize a DynamoDBMetadataStore instance "
- + "against the local DynamoDB server. Perhaps the DynamoDBLocal "
- + "server is not configured correctly. ";
- LOG.error(msg, e);
- // fail fast if the DynamoDBLocal server can not work
- throw e;
- }
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- if (dynamoDB != null) {
- dynamoDB.shutdown();
- }
- DynamoDBLocalClientFactory.stopSingletonServer();
- }
-
- /**
- * Each contract has its own S3AFileSystem and DynamoDBMetadataStore objects.
- */
- private static class DynamoDBMSContract extends AbstractMSContract {
- private final S3AFileSystem s3afs;
- private final DynamoDBMetadataStore ms = new DynamoDBMetadataStore();
-
- DynamoDBMSContract() throws IOException {
- this(new Configuration());
- }
-
- DynamoDBMSContract(Configuration conf) throws IOException {
- // using mocked S3 clients
- conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class,
- S3ClientFactory.class);
- conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, S3URI);
- // setting config for creating a DynamoDBClient against local server
- conf.set(ACCESS_KEY, "dummy-access-key");
- conf.set(SECRET_KEY, "dummy-secret-key");
- conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, true);
- conf.setClass(S3Guard.S3GUARD_DDB_CLIENT_FACTORY_IMPL,
- DynamoDBLocalClientFactory.class, DynamoDBClientFactory.class);
-
- // always create new file system object for a test contract
- s3afs = (S3AFileSystem) FileSystem.newInstance(conf);
- ms.initialize(s3afs);
- }
-
- @Override
- public S3AFileSystem getFileSystem() {
- return s3afs;
- }
-
- @Override
- public DynamoDBMetadataStore getMetadataStore() {
- return ms;
- }
- }
-
- @Override
- public DynamoDBMSContract createContract() throws IOException {
- return new DynamoDBMSContract();
- }
-
- @Override
- public DynamoDBMSContract createContract(Configuration conf) throws
- IOException {
- return new DynamoDBMSContract(conf);
- }
-
- @Override
- FileStatus basicFileStatus(Path path, int size, boolean isDir)
- throws IOException {
- String owner = UserGroupInformation.getCurrentUser().getShortUserName();
- return isDir
- ? new S3AFileStatus(true, path, owner)
- : new S3AFileStatus(size, getModTime(), path, BLOCK_SIZE, owner);
- }
-
- private DynamoDBMetadataStore getDynamoMetadataStore() throws IOException {
- return (DynamoDBMetadataStore) getContract().getMetadataStore();
- }
-
- private S3AFileSystem getFileSystem() throws IOException {
- return (S3AFileSystem) getContract().getFileSystem();
- }
-
- /**
- * This tests that after initialize() using an S3AFileSystem object, the
- * instance should have been initialized successfully, and tables are ACTIVE.
- */
- @Test
- public void testInitialize() throws IOException {
- final String tableName = "testInitializeWithFileSystem";
- final S3AFileSystem s3afs = getFileSystem();
- final Configuration conf = s3afs.getConf();
- conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
- try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
- ddbms.initialize(s3afs);
- verifyTableInitialized(tableName);
- assertNotNull(ddbms.getTable());
- assertEquals(tableName, ddbms.getTable().getTableName());
- String expectedRegion = conf.get(S3GUARD_DDB_REGION_KEY,
- s3afs.getBucketLocation(tableName));
- assertEquals("DynamoDB table should be in configured region or the same" +
- " region as S3 bucket",
- expectedRegion,
- ddbms.getRegion());
- }
- }
-
- /**
- * This tests that after initialize() using a Configuration object, the
- * instance should have been initialized successfully, and tables are ACTIVE.
- */
- @Test
- public void testInitializeWithConfiguration() throws IOException {
- final String tableName = "testInitializeWithConfiguration";
- final Configuration conf = getFileSystem().getConf();
- conf.unset(S3GUARD_DDB_TABLE_NAME_KEY);
- String savedRegion = conf.get(S3GUARD_DDB_REGION_KEY,
- getFileSystem().getBucketLocation());
- conf.unset(S3GUARD_DDB_REGION_KEY);
- try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
- ddbms.initialize(conf);
- fail("Should have failed because the table name is not set!");
- } catch (IllegalArgumentException ignored) {
- }
- // config table name
- conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
- try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
- ddbms.initialize(conf);
- fail("Should have failed because as the region is not set!");
- } catch (IllegalArgumentException ignored) {
- }
- // config region
- conf.set(S3GUARD_DDB_REGION_KEY, savedRegion);
- try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
- ddbms.initialize(conf);
- verifyTableInitialized(tableName);
- assertNotNull(ddbms.getTable());
- assertEquals(tableName, ddbms.getTable().getTableName());
- assertEquals("Unexpected key schema found!",
- keySchema(),
- ddbms.getTable().describe().getKeySchema());
- }
- }
-
- /**
- * Test that for a large batch write request, the limit is handled correctly.
- */
- @Test
- public void testBatchWrite() throws IOException {
- final int[] numMetasToDeleteOrPut = {
- -1, // null
- 0, // empty collection
- 1, // one path
- S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT, // exact limit of a batch request
- S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT + 1 // limit + 1
- };
- for (int numOldMetas : numMetasToDeleteOrPut) {
- for (int numNewMetas : numMetasToDeleteOrPut) {
- doTestBatchWrite(numOldMetas, numNewMetas);
- }
- }
- }
-
- private void doTestBatchWrite(int numDelete, int numPut) throws IOException {
- final String root = S3URI + "/testBatchWrite_" + numDelete + '_' + numPut;
- final Path oldDir = new Path(root, "oldDir");
- final Path newDir = new Path(root, "newDir");
- LOG.info("doTestBatchWrite: oldDir={}, newDir={}", oldDir, newDir);
-
- DynamoDBMetadataStore ms = getDynamoMetadataStore();
- ms.put(new PathMetadata(basicFileStatus(oldDir, 0, true)));
- ms.put(new PathMetadata(basicFileStatus(newDir, 0, true)));
-
- final List<PathMetadata> oldMetas =
- numDelete < 0 ? null : new ArrayList<PathMetadata>(numDelete);
- for (int i = 0; i < numDelete; i++) {
- oldMetas.add(new PathMetadata(
- basicFileStatus(new Path(oldDir, "child" + i), i, true)));
- }
- final List<PathMetadata> newMetas =
- numPut < 0 ? null : new ArrayList<PathMetadata>(numPut);
- for (int i = 0; i < numPut; i++) {
- newMetas.add(new PathMetadata(
- basicFileStatus(new Path(newDir, "child" + i), i, false)));
- }
-
- Collection<Path> pathsToDelete = null;
- if (oldMetas != null) {
- // put all metadata of old paths and verify
- ms.put(new DirListingMetadata(oldDir, oldMetas, false));
- assertEquals(0, ms.listChildren(newDir).withoutTombstones().numEntries());
- assertTrue(CollectionUtils.isEqualCollection(oldMetas,
- ms.listChildren(oldDir).getListing()));
-
- pathsToDelete = new ArrayList<>(oldMetas.size());
- for (PathMetadata meta : oldMetas) {
- pathsToDelete.add(meta.getFileStatus().getPath());
- }
- }
-
- // move the old paths to new paths and verify
- ms.move(pathsToDelete, newMetas);
- assertEquals(0, ms.listChildren(oldDir).withoutTombstones().numEntries());
- if (newMetas != null) {
- assertTrue(CollectionUtils.isEqualCollection(newMetas,
- ms.listChildren(newDir).getListing()));
- }
- }
-
- @Test
- public void testInitExistingTable() throws IOException {
- final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
- final String tableName = ddbms.getTable().getTableName();
- verifyTableInitialized(tableName);
- // create existing table
- ddbms.initTable();
- verifyTableInitialized(tableName);
- }
-
- /**
- * Test the low level version check code.
- */
- @Test
- public void testItemVersionCompatibility() throws Throwable {
- verifyVersionCompatibility("table",
- createVersionMarker(VERSION_MARKER, VERSION, 0));
- }
-
- /**
- * Test that a version marker entry without the version number field
- * is rejected as incompatible with a meaningful error message.
- */
- @Test
- public void testItemLacksVersion() throws Throwable {
- intercept(IOException.class, E_NOT_VERSION_MARKER,
- new VoidCallable() {
- @Override
- public void call() throws Exception {
- verifyVersionCompatibility("table",
- new Item().withPrimaryKey(
- createVersionMarkerPrimaryKey(VERSION_MARKER)));
- }
- });
- }
-
- /**
- * Delete the version marker and verify that table init fails.
- */
- @Test
- public void testTableVersionRequired() throws Exception {
- Configuration conf = getFileSystem().getConf();
- int maxRetries = conf.getInt(S3GUARD_DDB_MAX_RETRIES,
- S3GUARD_DDB_MAX_RETRIES_DEFAULT);
- conf.setInt(S3GUARD_DDB_MAX_RETRIES, 3);
-
- final DynamoDBMetadataStore ddbms = createContract(conf).getMetadataStore();
- String tableName = conf.get(S3GUARD_DDB_TABLE_NAME_KEY, BUCKET);
- Table table = verifyTableInitialized(tableName);
- table.deleteItem(VERSION_MARKER_PRIMARY_KEY);
-
- // create existing table
- intercept(IOException.class, E_NO_VERSION_MARKER,
- new VoidCallable() {
- @Override
- public void call() throws Exception {
- ddbms.initTable();
- }
- });
-
- conf.setInt(S3GUARD_DDB_MAX_RETRIES, maxRetries);
- }
-
- /**
- * Set the version value to a different number and verify that
- * table init fails.
- */
- @Test
- public void testTableVersionMismatch() throws Exception {
- final DynamoDBMetadataStore ddbms = createContract().getMetadataStore();
- String tableName = getFileSystem().getConf()
- .get(S3GUARD_DDB_TABLE_NAME_KEY, BUCKET);
- Table table = verifyTableInitialized(tableName);
- table.deleteItem(VERSION_MARKER_PRIMARY_KEY);
- Item v200 = createVersionMarker(VERSION_MARKER, 200, 0);
- table.putItem(v200);
-
- // create existing table
- intercept(IOException.class, E_INCOMPATIBLE_VERSION,
- new VoidCallable() {
- @Override
- public void call() throws Exception {
- ddbms.initTable();
- }
- });
- }
-
- /**
- * Test that initTable fails with IOException when table does not exist and
- * table auto-creation is disabled.
- */
- @Test
- public void testFailNonexistentTable() throws IOException {
- final String tableName = "testFailNonexistentTable";
- final S3AFileSystem s3afs = getFileSystem();
- final Configuration conf = s3afs.getConf();
- conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
- conf.unset(S3GUARD_DDB_TABLE_CREATE_KEY);
- try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
- ddbms.initialize(s3afs);
- fail("Should have failed as table does not exist and table auto-creation"
- + " is disabled");
- } catch (IOException ignored) {
- }
- }
-
- /**
- * Test cases about root directory as it is not in the DynamoDB table.
- */
- @Test
- public void testRootDirectory() throws IOException {
- final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
- Path rootPath = new Path(S3URI);
- verifyRootDirectory(ddbms.get(rootPath), true);
-
- ddbms.put(new PathMetadata(new S3AFileStatus(true,
- new Path(rootPath, "foo"),
- UserGroupInformation.getCurrentUser().getShortUserName())));
- verifyRootDirectory(ddbms.get(new Path(S3URI)), false);
- }
-
- private void verifyRootDirectory(PathMetadata rootMeta, boolean isEmpty) {
- assertNotNull(rootMeta);
- final FileStatus status = rootMeta.getFileStatus();
- assertNotNull(status);
- assertTrue(status.isDirectory());
- // UNKNOWN is always a valid option, but true / false should not contradict
- if (isEmpty) {
- assertNotSame("Should not be marked non-empty",
- Tristate.FALSE,
- rootMeta.isEmptyDirectory());
- } else {
- assertNotSame("Should not be marked empty",
- Tristate.TRUE,
- rootMeta.isEmptyDirectory());
- }
- }
-
- /**
- * Test that when moving nested paths, all its ancestors up to destination
- * root will also be created.
- * Here is the directory tree before move:
- * <pre>
- * testMovePopulateAncestors
- * ├── a
- * │  └── b
- * │  └── src
- * │  ├── dir1
- * │  │  └── dir2
- * │  └── file1.txt
- * └── c
- * └── d
- * └── dest
- *</pre>
- * As part of rename(a/b/src, d/c/dest), S3A will enumerate the subtree at
- * a/b/src. This test verifies that after the move, the new subtree at
- * 'dest' is reachable from the root (i.e. c/ and c/d exist in the table.
- * DynamoDBMetadataStore depends on this property to do recursive delete
- * without a full table scan.
- */
- @Test
- public void testMovePopulatesAncestors() throws IOException {
- final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
- final String testRoot = "/testMovePopulatesAncestors";
- final String srcRoot = testRoot + "/a/b/src";
- final String destRoot = testRoot + "/c/d/e/dest";
-
- final Path nestedPath1 = strToPath(srcRoot + "/file1.txt");
- ddbms.put(new PathMetadata(basicFileStatus(nestedPath1, 1024, false)));
- final Path nestedPath2 = strToPath(srcRoot + "/dir1/dir2");
- ddbms.put(new PathMetadata(basicFileStatus(nestedPath2, 0, true)));
-
- // We don't put the destRoot path here, since put() would create ancestor
- // entries, and we want to ensure that move() does it, instead.
-
- // Build enumeration of src / dest paths and do the move()
- final Collection<Path> fullSourcePaths = Lists.newArrayList(
- strToPath(srcRoot),
- strToPath(srcRoot + "/dir1"),
- strToPath(srcRoot + "/dir1/dir2"),
- strToPath(srcRoot + "/file1.txt")
- );
- final Collection<PathMetadata> pathsToCreate = Lists.newArrayList(
- new PathMetadata(basicFileStatus(strToPath(destRoot),
- 0, true)),
- new PathMetadata(basicFileStatus(strToPath(destRoot + "/dir1"),
- 0, true)),
- new PathMetadata(basicFileStatus(strToPath(destRoot + "/dir1/dir2"),
- 0, true)),
- new PathMetadata(basicFileStatus(strToPath(destRoot + "/file1.txt"),
- 1024, false))
- );
-
- ddbms.move(fullSourcePaths, pathsToCreate);
-
- // assert that all the ancestors should have been populated automatically
- assertCached(testRoot + "/c");
- assertCached(testRoot + "/c/d");
- assertCached(testRoot + "/c/d/e");
- assertCached(destRoot /* /c/d/e/dest */);
-
- // Also check moved files while we're at it
- assertCached(destRoot + "/dir1");
- assertCached(destRoot + "/dir1/dir2");
- assertCached(destRoot + "/file1.txt");
- }
-
- @Test
- public void testProvisionTable() throws IOException {
- final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
- final String tableName = ddbms.getTable().getTableName();
- final ProvisionedThroughputDescription oldProvision =
- dynamoDB.getTable(tableName).describe().getProvisionedThroughput();
- ddbms.provisionTable(oldProvision.getReadCapacityUnits() * 2,
- oldProvision.getWriteCapacityUnits() * 2);
- final ProvisionedThroughputDescription newProvision =
- dynamoDB.getTable(tableName).describe().getProvisionedThroughput();
- LOG.info("Old provision = {}, new provision = {}",
- oldProvision, newProvision);
- assertEquals(oldProvision.getReadCapacityUnits() * 2,
- newProvision.getReadCapacityUnits().longValue());
- assertEquals(oldProvision.getWriteCapacityUnits() * 2,
- newProvision.getWriteCapacityUnits().longValue());
- }
-
- @Test
- public void testDeleteTable() throws Exception {
- final String tableName = "testDeleteTable";
- final S3AFileSystem s3afs = getFileSystem();
- final Configuration conf = s3afs.getConf();
- conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
- try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
- ddbms.initialize(s3afs);
- // we can list the empty table
- ddbms.listChildren(new Path(S3URI));
-
- ddbms.destroy();
- verifyTableNotExist(tableName);
-
- // delete table once more; be ResourceNotFoundException swallowed silently
- ddbms.destroy();
- verifyTableNotExist(tableName);
- try {
- // we can no longer list the destroyed table
- ddbms.listChildren(new Path(S3URI));
- fail("Should have failed after the table is destroyed!");
- } catch (IOException ignored) {
- }
- }
- }
-
- /**
- * This validates the table is created and ACTIVE in DynamoDB.
- *
- * This should not rely on the {@link DynamoDBMetadataStore} implementation.
- * Return the table
- */
- private static Table verifyTableInitialized(String tableName) {
- final Table table = dynamoDB.getTable(tableName);
- final TableDescription td = table.describe();
- assertEquals(tableName, td.getTableName());
- assertEquals("ACTIVE", td.getTableStatus());
- return table;
- }
-
- /**
- * This validates the table is not found in DynamoDB.
- *
- * This should not rely on the {@link DynamoDBMetadataStore} implementation.
- */
- private static void verifyTableNotExist(String tableName) throws Exception{
- intercept(ResourceNotFoundException.class,
- () -> dynamoDB.getTable(tableName).describe());
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-7449. Split up class
TestYarnClient to TestYarnClient and TestYarnClientImpl. Contributed by
Szilard Nemeth.
Posted by xk...@apache.org.
YARN-7449. Split up class TestYarnClient to TestYarnClient and TestYarnClientImpl. Contributed by Szilard Nemeth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbbc7cc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbbc7cc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbbc7cc4
Branch: refs/heads/HDFS-12943
Commit: bbbc7cc426f71ad0fe4174efcd25e5ac3f62b501
Parents: 55432b0
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Wed Jun 20 11:40:56 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Wed Jun 20 11:40:56 2018 -0700
----------------------------------------------------------------------
.../yarn/client/api/impl/TestYarnClient.java | 337 ++++---------------
.../client/api/impl/TestYarnClientImpl.java | 254 ++++++++++++++
2 files changed, 324 insertions(+), 267 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbbc7cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 70ff47b..17e43ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -18,41 +18,9 @@
package org.apache.hadoop.yarn.client.api.impl;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.lang.Thread.State;
-import java.nio.ByteBuffer;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.DataInputByteBuffer;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
@@ -74,7 +42,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -92,7 +59,6 @@ import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.AHSClient;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -100,7 +66,6 @@ import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase;
@@ -115,8 +80,28 @@ import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.slf4j.event.Level;
+import java.io.IOException;
+import java.lang.Thread.State;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
/**
- * This class is to test class {@link YarnClient) and {@link YarnClientImpl}.
+ * This class is to test class {@link YarnClient).
*/
public class TestYarnClient extends ParameterizedSchedulerTestBase {
@@ -147,17 +132,6 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
}
@Test
- public void testStartWithTimelineV15() throws Exception {
- Configuration conf = getConf();
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
- YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient();
- client.init(conf);
- client.start();
- client.stop();
- }
-
- @Test
public void testStartTimelineClientWithErrors()
throws Exception {
// If timeline client failed to init with a NoClassDefFoundError
@@ -413,7 +387,7 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
RMApp app = rm.submitApp(2000);
RMApp app1 =
rm.submitApp(200, "name", "user",
- new HashMap<ApplicationAccessType, String>(), false, "default", -1,
+ new HashMap<>(), false, "default", -1,
null, "MAPREDUCE");
Assert.assertEquals("YARN", app.getApplicationType());
Assert.assertEquals("MAPREDUCE", app1.getApplicationType());
@@ -427,7 +401,7 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
rm.start();
RMApp app1 =
rm.submitApp(200, "name", "user",
- new HashMap<ApplicationAccessType, String>(), false, "default", -1,
+ new HashMap<>(), false, "default", -1,
null, "MAPREDUCE-LENGTH-IS-20");
Assert.assertEquals("MAPREDUCE-LENGTH-IS-", app1.getApplicationType());
rm.stop();
@@ -444,7 +418,7 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
List<ApplicationReport> reports = client.getApplications();
Assert.assertEquals(reports, expectedReports);
- Set<String> appTypes = new HashSet<String>();
+ Set<String> appTypes = new HashSet<>();
appTypes.add("YARN");
appTypes.add("NON-YARN");
@@ -601,7 +575,7 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
Assert.assertEquals(labelsToNodes.size(), 3);
// Get labels to nodes for selected labels
- Set<String> setLabels = new HashSet<String>(Arrays.asList("x", "z"));
+ Set<String> setLabels = new HashSet<>(Arrays.asList("x", "z"));
expectedLabelsToNodes =
((MockYarnClient)client).getLabelsToNodesMap(setLabels);
labelsToNodes = client.getLabelsToNodes(setLabels);
@@ -633,12 +607,12 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
private ApplicationReport mockReport;
private List<ApplicationReport> reports;
- private HashMap<ApplicationId, List<ApplicationAttemptReport>> attempts =
- new HashMap<ApplicationId, List<ApplicationAttemptReport>>();
- private HashMap<ApplicationAttemptId, List<ContainerReport>> containers =
- new HashMap<ApplicationAttemptId, List<ContainerReport>>();
+ private HashMap<ApplicationId, List<ApplicationAttemptReport>> attempts =
+ new HashMap<>();
+ private HashMap<ApplicationAttemptId, List<ContainerReport>> containers =
+ new HashMap<>();
private HashMap<ApplicationAttemptId, List<ContainerReport>> containersFromAHS =
- new HashMap<ApplicationAttemptId, List<ContainerReport>>();
+ new HashMap<>();
GetApplicationsResponse mockAppResponse =
mock(GetApplicationsResponse.class);
@@ -739,9 +713,9 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
- List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
+ List<ApplicationReport> applicationReports = new ArrayList<>();
applicationReports.add(newApplicationReport);
- List<ApplicationAttemptReport> appAttempts = new ArrayList<ApplicationAttemptReport>();
+ List<ApplicationAttemptReport> appAttempts = new ArrayList<>();
ApplicationAttemptReport attempt = ApplicationAttemptReport.newInstance(
ApplicationAttemptId.newInstance(applicationId, 1),
"host",
@@ -767,7 +741,7 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
appAttempts.add(attempt1);
attempts.put(applicationId, appAttempts);
- List<ContainerReport> containerReports = new ArrayList<ContainerReport>();
+ List<ContainerReport> containerReports = new ArrayList<>();
ContainerReport container = ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1), null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
@@ -785,7 +759,7 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
//add containers to be sent from AHS
List<ContainerReport> containerReportsForAHS =
- new ArrayList<ContainerReport>();
+ new ArrayList<>();
container = ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1), null,
@@ -843,7 +817,7 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
List<ApplicationReport> applicationReports,
Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates) {
- List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
+ List<ApplicationReport> appReports = new ArrayList<>();
for (ApplicationReport appReport : applicationReports) {
if (applicationTypes != null && !applicationTypes.isEmpty()) {
if (!applicationTypes.contains(appReport.getApplicationType())) {
@@ -878,9 +852,9 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
}
public Map<String, Set<NodeId>> getLabelsToNodesMap() {
- Map<String, Set<NodeId>> map = new HashMap<String, Set<NodeId>>();
+ Map<String, Set<NodeId>> map = new HashMap<>();
Set<NodeId> setNodeIds =
- new HashSet<NodeId>(Arrays.asList(
+ new HashSet<>(Arrays.asList(
NodeId.newInstance("host1", 0), NodeId.newInstance("host2", 0)));
map.put("x", setNodeIds);
map.put("y", setNodeIds);
@@ -889,8 +863,8 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
}
public Map<String, Set<NodeId>> getLabelsToNodesMap(Set<String> labels) {
- Map<String, Set<NodeId>> map = new HashMap<String, Set<NodeId>>();
- Set<NodeId> setNodeIds = new HashSet<NodeId>(Arrays.asList(
+ Map<String, Set<NodeId>> map = new HashMap<>();
+ Set<NodeId> setNodeIds = new HashSet<>(Arrays.asList(
NodeId.newInstance("host1", 0), NodeId.newInstance("host2", 0)));
for (String label : labels) {
map.put(label, setNodeIds);
@@ -907,8 +881,8 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
}
public Map<NodeId, Set<String>> getNodeToLabelsMap() {
- Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
- Set<String> setNodeLabels = new HashSet<String>(Arrays.asList("x", "y"));
+ Map<NodeId, Set<String>> map = new HashMap<>();
+ Set<String> setNodeLabels = new HashSet<>(Arrays.asList("x", "y"));
map.put(NodeId.newInstance("host", 0), setNodeLabels);
return map;
}
@@ -985,7 +959,7 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
private ContainerReport getContainer(
ContainerId containerId,
HashMap<ApplicationAttemptId, List<ContainerReport>> containersToAppAttemptMapping)
- throws YarnException, IOException {
+ throws YarnException {
List<ContainerReport> containersForAppAttempt =
containersToAppAttemptMapping.get(containerId
.getApplicationAttemptId());
@@ -1119,174 +1093,6 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
Assert.assertEquals(unmanagedApplication, report.isUnmanagedApp());
}
- @Test
- public void testAsyncAPIPollTimeout() {
- testAsyncAPIPollTimeoutHelper(null, false);
- testAsyncAPIPollTimeoutHelper(0L, true);
- testAsyncAPIPollTimeoutHelper(1L, true);
- }
-
- private void testAsyncAPIPollTimeoutHelper(Long valueForTimeout,
- boolean expectedTimeoutEnforcement) {
- YarnClientImpl client = new YarnClientImpl();
- try {
- Configuration conf = getConf();
- if (valueForTimeout != null) {
- conf.setLong(
- YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS,
- valueForTimeout);
- }
-
- client.init(conf);
-
- Assert.assertEquals(
- expectedTimeoutEnforcement, client.enforceAsyncAPITimeout());
- } finally {
- IOUtils.closeQuietly(client);
- }
- }
-
- @Test
- public void testBestEffortTimelineDelegationToken()
- throws Exception {
- Configuration conf = getConf();
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
-
- YarnClientImpl client = spy(new YarnClientImpl() {
-
- @Override
- TimelineClient createTimelineClient() throws IOException, YarnException {
- timelineClient = mock(TimelineClient.class);
- when(timelineClient.getDelegationToken(any(String.class)))
- .thenThrow(new RuntimeException("Best effort test exception"));
- return timelineClient;
- }
- });
-
- client.init(conf);
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
- true);
- client.serviceInit(conf);
- client.getTimelineDelegationToken();
-
- try {
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, false);
- client.serviceInit(conf);
- client.getTimelineDelegationToken();
- Assert.fail("Get delegation token should have thrown an exception");
- } catch (IOException e) {
- // Success
- }
- }
-
- @Test
- public void testAutomaticTimelineDelegationTokenLoading()
- throws Exception {
- Configuration conf = getConf();
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
- TimelineDelegationTokenIdentifier timelineDT =
- new TimelineDelegationTokenIdentifier();
- final Token<TimelineDelegationTokenIdentifier> dToken =
- new Token<TimelineDelegationTokenIdentifier>(
- timelineDT.getBytes(), new byte[0], timelineDT.getKind(), new Text());
- // create a mock client
- YarnClientImpl client = spy(new YarnClientImpl() {
-
- @Override
- TimelineClient createTimelineClient() throws IOException, YarnException {
- timelineClient = mock(TimelineClient.class);
- when(timelineClient.getDelegationToken(any(String.class)))
- .thenReturn(dToken);
- return timelineClient;
- }
-
-
- @Override
- protected void serviceStart() throws Exception {
- rmClient = mock(ApplicationClientProtocol.class);
- }
-
- @Override
- protected void serviceStop() throws Exception {
- }
-
- @Override
- public ApplicationReport getApplicationReport(ApplicationId appId) {
- ApplicationReport report = mock(ApplicationReport.class);
- when(report.getYarnApplicationState())
- .thenReturn(YarnApplicationState.RUNNING);
- return report;
- }
-
- @Override
- public boolean isSecurityEnabled() {
- return true;
- }
- });
- client.init(conf);
- client.start();
- try {
- // when i == 0, timeline DT already exists, no need to get one more
- // when i == 1, timeline DT doesn't exist, need to get one more
- for (int i = 0; i < 2; ++i) {
- ApplicationSubmissionContext context =
- mock(ApplicationSubmissionContext.class);
- ApplicationId applicationId = ApplicationId.newInstance(0, i + 1);
- when(context.getApplicationId()).thenReturn(applicationId);
- DataOutputBuffer dob = new DataOutputBuffer();
- Credentials credentials = new Credentials();
- if (i == 0) {
- credentials.addToken(client.timelineService, dToken);
- }
- credentials.writeTokenStorageToStream(dob);
- ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- ContainerLaunchContext clc = ContainerLaunchContext.newInstance(
- null, null, null, null, tokens, null);
- when(context.getAMContainerSpec()).thenReturn(clc);
- client.submitApplication(context);
- if (i == 0) {
- // GetTimelineDelegationToken shouldn't be called
- verify(client, never()).getTimelineDelegationToken();
- }
- // In either way, token should be there
- credentials = new Credentials();
- DataInputByteBuffer dibb = new DataInputByteBuffer();
- tokens = clc.getTokens();
- if (tokens != null) {
- dibb.reset(tokens);
- credentials.readTokenStorageStream(dibb);
- tokens.rewind();
- }
- Collection<Token<? extends TokenIdentifier>> dTokens =
- credentials.getAllTokens();
- Assert.assertEquals(1, dTokens.size());
- Assert.assertEquals(dToken, dTokens.iterator().next());
- }
- } finally {
- client.stop();
- }
- }
-
- @Test
- public void testParseTimelineDelegationTokenRenewer() throws Exception {
- // Client side
- YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient();
- Configuration conf = getConf();
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- conf.set(YarnConfiguration.RM_PRINCIPAL, "rm/_HOST@EXAMPLE.COM");
- conf.set(
- YarnConfiguration.RM_ADDRESS, "localhost:8188");
- try {
- client.init(conf);
- client.start();
- Assert.assertEquals("rm/localhost@EXAMPLE.COM", client.timelineDTRenewer);
- } finally {
- client.stop();
- }
- }
-
@Test(timeout = 30000, expected = ApplicationNotFoundException.class)
public void testShouldNotRetryForeverForNonNetworkExceptions() throws Exception {
YarnConfiguration conf = getConf();
@@ -1353,38 +1159,35 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
timelineClientBestEffort);
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION,
timelineVersion);
- YarnClient client = new MockYarnClient();
- if (client instanceof YarnClientImpl) {
- YarnClientImpl impl = (YarnClientImpl) client;
- YarnClientImpl spyClient = spy(impl);
- when(spyClient.createTimelineClient()).thenThrow(mockErr);
- CreateTimelineClientErrorVerifier verifier = spy(errVerifier);
- spyClient.init(conf);
- spyClient.start();
-
- ApplicationSubmissionContext context =
- mock(ApplicationSubmissionContext.class);
- ContainerLaunchContext containerContext =
- mock(ContainerLaunchContext.class);
- ApplicationId applicationId =
- ApplicationId.newInstance(System.currentTimeMillis(), 1);
- when(containerContext.getTokens()).thenReturn(null);
- when(context.getApplicationId()).thenReturn(applicationId);
- when(spyClient.isSecurityEnabled()).thenReturn(true);
- when(context.getAMContainerSpec()).thenReturn(containerContext);
+ MockYarnClient client = new MockYarnClient();
+ MockYarnClient spyClient = spy(client);
+ when(spyClient.createTimelineClient()).thenThrow(mockErr);
+ CreateTimelineClientErrorVerifier verifier = spy(errVerifier);
+ spyClient.init(conf);
+ spyClient.start();
+
+ ApplicationSubmissionContext context =
+ mock(ApplicationSubmissionContext.class);
+ ContainerLaunchContext containerContext =
+ mock(ContainerLaunchContext.class);
+ ApplicationId applicationId =
+ ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ when(containerContext.getTokens()).thenReturn(null);
+ when(context.getApplicationId()).thenReturn(applicationId);
+ when(spyClient.isSecurityEnabled()).thenReturn(true);
+ when(context.getAMContainerSpec()).thenReturn(containerContext);
- try {
- spyClient.submitApplication(context);
- } catch (Throwable e) {
- verifier.verifyError(e);
- } finally {
- // Make sure the verifier runs with expected times
- // This is required because in case throwable is swallowed
- // and verifyError never gets the chance to run
- verify(verifier, times(verifier.getExpectedTimes()))
- .verifyError(any(Throwable.class));
- spyClient.stop();
- }
+ try {
+ spyClient.submitApplication(context);
+ } catch (Throwable e) {
+ verifier.verifyError(e);
+ } finally {
+ // Make sure the verifier runs with expected times
+ // This is required because in case throwable is swallowed
+ // and verifyError never gets the chance to run
+ verify(verifier, times(verifier.getExpectedTimes()))
+ .verifyError(any(Throwable.class));
+ spyClient.stop();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbbc7cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientImpl.java
new file mode 100644
index 0000000..dd0aa5c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientImpl.java
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api.impl;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager
+ .ParameterizedSchedulerTestBase;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * This class is to test class {@link YarnClientImpl ).
+ */
+public class TestYarnClientImpl extends ParameterizedSchedulerTestBase {
+
+ public TestYarnClientImpl(SchedulerType type) throws IOException {
+ super(type);
+ }
+
+ @Before
+ public void setup() {
+ QueueMetrics.clearQueueMetrics();
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ }
+
+ @Test
+ public void testStartWithTimelineV15() {
+ Configuration conf = getConf();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
+ YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient();
+ client.init(conf);
+ client.start();
+ client.stop();
+ }
+
+ @Test
+ public void testAsyncAPIPollTimeout() {
+ testAsyncAPIPollTimeoutHelper(null, false);
+ testAsyncAPIPollTimeoutHelper(0L, true);
+ testAsyncAPIPollTimeoutHelper(1L, true);
+ }
+
+ private void testAsyncAPIPollTimeoutHelper(Long valueForTimeout,
+ boolean expectedTimeoutEnforcement) {
+ YarnClientImpl client = new YarnClientImpl();
+ try {
+ Configuration conf = getConf();
+ if (valueForTimeout != null) {
+ conf.setLong(
+ YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS,
+ valueForTimeout);
+ }
+
+ client.init(conf);
+
+ Assert.assertEquals(
+ expectedTimeoutEnforcement, client.enforceAsyncAPITimeout());
+ } finally {
+ IOUtils.closeQuietly(client);
+ }
+ }
+
+ @Test
+ public void testBestEffortTimelineDelegationToken()
+ throws Exception {
+ Configuration conf = getConf();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
+
+ YarnClientImpl client = spy(new YarnClientImpl() {
+
+ @Override
+ TimelineClient createTimelineClient() throws IOException, YarnException {
+ timelineClient = mock(TimelineClient.class);
+ when(timelineClient.getDelegationToken(any(String.class)))
+ .thenThrow(new RuntimeException("Best effort test exception"));
+ return timelineClient;
+ }
+ });
+
+ client.init(conf);
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
+ true);
+ client.serviceInit(conf);
+ client.getTimelineDelegationToken();
+
+ try {
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, false);
+ client.serviceInit(conf);
+ client.getTimelineDelegationToken();
+ Assert.fail("Get delegation token should have thrown an exception");
+ } catch (IOException e) {
+ // Success
+ }
+ }
+
+ @Test
+ public void testAutomaticTimelineDelegationTokenLoading()
+ throws Exception {
+ Configuration conf = getConf();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
+ TimelineDelegationTokenIdentifier timelineDT =
+ new TimelineDelegationTokenIdentifier();
+ final Token<TimelineDelegationTokenIdentifier> dToken =
+ new Token<>(
+ timelineDT.getBytes(), new byte[0], timelineDT.getKind(), new Text());
+ // create a mock client
+ YarnClientImpl client = spy(new YarnClientImpl() {
+
+ @Override
+ TimelineClient createTimelineClient() throws IOException, YarnException {
+ timelineClient = mock(TimelineClient.class);
+ when(timelineClient.getDelegationToken(any(String.class)))
+ .thenReturn(dToken);
+ return timelineClient;
+ }
+
+
+ @Override
+ protected void serviceStart() {
+ rmClient = mock(ApplicationClientProtocol.class);
+ }
+
+ @Override
+ protected void serviceStop() {
+ }
+
+ @Override
+ public ApplicationReport getApplicationReport(ApplicationId appId) {
+ ApplicationReport report = mock(ApplicationReport.class);
+ when(report.getYarnApplicationState())
+ .thenReturn(YarnApplicationState.RUNNING);
+ return report;
+ }
+
+ @Override
+ public boolean isSecurityEnabled() {
+ return true;
+ }
+ });
+ client.init(conf);
+ client.start();
+ try {
+ // when i == 0, timeline DT already exists, no need to get one more
+ // when i == 1, timeline DT doesn't exist, need to get one more
+ for (int i = 0; i < 2; ++i) {
+ ApplicationSubmissionContext context =
+ mock(ApplicationSubmissionContext.class);
+ ApplicationId applicationId = ApplicationId.newInstance(0, i + 1);
+ when(context.getApplicationId()).thenReturn(applicationId);
+ DataOutputBuffer dob = new DataOutputBuffer();
+ Credentials credentials = new Credentials();
+ if (i == 0) {
+ credentials.addToken(client.timelineService, dToken);
+ }
+ credentials.writeTokenStorageToStream(dob);
+ ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ ContainerLaunchContext clc = ContainerLaunchContext.newInstance(
+ null, null, null, null, tokens, null);
+ when(context.getAMContainerSpec()).thenReturn(clc);
+ client.submitApplication(context);
+ if (i == 0) {
+ // GetTimelineDelegationToken shouldn't be called
+ verify(client, never()).getTimelineDelegationToken();
+ }
+ // In either way, token should be there
+ credentials = new Credentials();
+ DataInputByteBuffer dibb = new DataInputByteBuffer();
+ tokens = clc.getTokens();
+ if (tokens != null) {
+ dibb.reset(tokens);
+ credentials.readTokenStorageStream(dibb);
+ tokens.rewind();
+ }
+ Collection<Token<? extends TokenIdentifier>> dTokens =
+ credentials.getAllTokens();
+ Assert.assertEquals(1, dTokens.size());
+ Assert.assertEquals(dToken, dTokens.iterator().next());
+ }
+ } finally {
+ client.stop();
+ }
+ }
+
+ @Test
+ public void testParseTimelineDelegationTokenRenewer() {
+ // Client side
+ YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient();
+ Configuration conf = getConf();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.set(YarnConfiguration.RM_PRINCIPAL, "rm/_HOST@EXAMPLE.COM");
+ conf.set(
+ YarnConfiguration.RM_ADDRESS, "localhost:8188");
+ try {
+ client.init(conf);
+ client.start();
+ Assert.assertEquals("rm/localhost@EXAMPLE.COM", client.timelineDTRenewer);
+ } finally {
+ client.stop();
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HDFS-13681. Fix
TestStartup.testNNFailToStartOnReadOnlyNNDir test failure on Windows.
Contributed by Xiao Liang.
Posted by xk...@apache.org.
HDFS-13681. Fix TestStartup.testNNFailToStartOnReadOnlyNNDir test failure on Windows. Contributed by Xiao Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8762e9cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8762e9cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8762e9cf
Branch: refs/heads/HDFS-12943
Commit: 8762e9cf10fa100dd5f7fd695f5e52b75a94c5d4
Parents: d31a3ce
Author: Inigo Goiri <in...@apache.org>
Authored: Fri Jun 15 16:49:06 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Fri Jun 15 16:49:06 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/namenode/TestStartup.java | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8762e9cf/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
index d5f5487..2401608 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
@@ -728,8 +728,8 @@ public class TestStartup {
assertTrue(nnDirs.iterator().hasNext());
assertEquals(
"NN dir should be created after NN startup.",
- nnDirStr,
- nnDirs.iterator().next().getPath());
+ new File(nnDirStr),
+ new File(nnDirs.iterator().next().getPath()));
final File nnDir = new File(nnDirStr);
assertTrue(nnDir.exists());
assertTrue(nnDir.isDirectory());
@@ -738,7 +738,7 @@ public class TestStartup {
/* set read only */
assertTrue(
"Setting NN dir read only should succeed.",
- nnDir.setReadOnly());
+ FileUtil.setWritable(nnDir, false));
cluster.restartNameNodes();
fail("Restarting NN should fail on read only NN dir.");
} catch (InconsistentFSStateException e) {
@@ -750,7 +750,8 @@ public class TestStartup {
"storage directory does not exist or is not accessible."))));
} finally {
/* set back to writable in order to clean it */
- assertTrue("Setting NN dir should succeed.", nnDir.setWritable(true));
+ assertTrue("Setting NN dir should succeed.",
+ FileUtil.setWritable(nnDir, true));
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDDS-191. Queue SCMCommands via
EventQueue in SCM. Contributed by Elek, Marton.
Posted by xk...@apache.org.
HDDS-191. Queue SCMCommands via EventQueue in SCM.
Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a55d6bba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a55d6bba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a55d6bba
Branch: refs/heads/HDFS-12943
Commit: a55d6bba71c81c1c4e9d8cd11f55c78f10a548b0
Parents: 4ffe68a
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Jun 25 13:05:22 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Jun 25 13:05:22 2018 -0700
----------------------------------------------------------------------
.../protocol/commands/CommandForDatanode.java | 45 ++++++++++++++++++++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 20 ++++++++-
.../scm/server/StorageContainerManager.java | 8 +++-
.../hadoop/hdds/scm/node/TestNodeManager.java | 39 +++++++++++++++++
4 files changed, 110 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d6bba/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
new file mode 100644
index 0000000..0c4964a
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.protocol.commands;
+
+import java.util.UUID;
+
+import com.google.protobuf.GeneratedMessage;
+
+/**
+ * Command for the datanode with the destination address.
+ */
+public class CommandForDatanode<T extends GeneratedMessage> {
+
+ private final UUID datanodeId;
+
+ private final SCMCommand<T> command;
+
+ public CommandForDatanode(UUID datanodeId, SCMCommand<T> command) {
+ this.datanodeId = datanodeId;
+ this.command = command;
+ }
+
+ public UUID getDatanodeId() {
+ return datanodeId;
+ }
+
+ public SCMCommand<T> getCommand() {
+ return command;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d6bba/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index b339fb7..fc8b013 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -25,6 +25,10 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.VersionInfo;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
+import org.apache.hadoop.hdds.server.events.Event;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.server.events.TypedEvent;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -42,11 +46,14 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
import org.apache.hadoop.ozone.protocol.VersionResponse;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
+
+import com.google.protobuf.GeneratedMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,7 +107,8 @@ import static org.apache.hadoop.util.Time.monotonicNow;
* as soon as you read it.
*/
public class SCMNodeManager
- implements NodeManager, StorageContainerNodeProtocol {
+ implements NodeManager, StorageContainerNodeProtocol,
+ EventHandler<CommandForDatanode> {
@VisibleForTesting
static final Logger LOG =
@@ -154,6 +162,9 @@ public class SCMNodeManager
private final SCMNodePoolManager nodePoolManager;
private final StorageContainerManager scmManager;
+ public static final Event<CommandForDatanode> DATANODE_COMMAND =
+ new TypedEvent<>(CommandForDatanode.class, "DATANODE_COMMAND");
+
/**
* Constructs SCM machine Manager.
*/
@@ -871,4 +882,11 @@ public class SCMNodeManager
public void setStaleNodeIntervalMs(long interval) {
this.staleNodeIntervalMs = interval;
}
+
+ @Override
+ public void onMessage(CommandForDatanode commandForDatanode,
+ EventPublisher publisher) {
+ addDatanodeCommand(commandForDatanode.getDatanodeId(),
+ commandForDatanode.getCommand());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d6bba/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 78f13cb..5725d23 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
+import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RPC;
@@ -51,6 +52,7 @@ import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.common.StorageInfo;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.StringUtils;
@@ -161,8 +163,12 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
throw new SCMException("SCM not initialized.", ResultCodes
.SCM_NOT_INITIALIZED);
}
+ EventQueue eventQueue = new EventQueue();
+
+ SCMNodeManager nm = new SCMNodeManager(conf, scmStorage.getClusterID(), this);
+ scmNodeManager = nm;
+ eventQueue.addHandler(SCMNodeManager.DATANODE_COMMAND, nm);
- scmNodeManager = new SCMNodeManager(conf, scmStorage.getClusterID(), this);
scmContainerManager = new ContainerMapping(conf, getScmNodeManager(),
cacheSize);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d6bba/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
index 2b04d6b..824a135 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.node;
import com.google.common.base.Supplier;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -29,7 +30,10 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.StorageReportProto;
+import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
@@ -1165,4 +1169,39 @@ public class TestNodeManager {
assertEquals(expectedRemaining, foundRemaining);
}
}
+
+ @Test
+ public void testHandlingSCMCommandEvent() {
+ OzoneConfiguration conf = getConf();
+ conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
+ 100, TimeUnit.MILLISECONDS);
+
+ DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
+ String dnId = datanodeDetails.getUuidString();
+ String storagePath = testDir.getAbsolutePath() + "/" + dnId;
+ List<StorageReportProto> reports =
+ TestUtils.createStorageReport(100, 10, 90,
+ storagePath, null, dnId, 1);
+
+ EventQueue eq = new EventQueue();
+ try (SCMNodeManager nodemanager = createNodeManager(conf)) {
+ eq.addHandler(SCMNodeManager.DATANODE_COMMAND, nodemanager);
+
+ nodemanager
+ .register(datanodeDetails, TestUtils.createNodeReport(reports));
+ eq.fireEvent(SCMNodeManager.DATANODE_COMMAND,
+ new CommandForDatanode(datanodeDetails.getUuid(),
+ new CloseContainerCommand(1L, ReplicationType.STAND_ALONE)));
+
+ eq.processAll(1000L);
+ List<SCMCommand> command =
+ nodemanager.sendHeartbeat(datanodeDetails, null);
+ Assert.assertEquals(1, command.size());
+ Assert
+ .assertEquals(command.get(0).getClass(), CloseContainerCommand.class);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HADOOP-15423. Merge fileCache and
dirCache into ine single cache in LocalMetadataStore. Contributed by Gabor
Bota.
Posted by xk...@apache.org.
HADOOP-15423. Merge fileCache and dirCache into ine single cache in LocalMetadataStore. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c687a661
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c687a661
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c687a661
Branch: refs/heads/HDFS-12943
Commit: c687a6617d73293019d8d91ac48bbfd2ccca3b40
Parents: a55d6bb
Author: Sean Mackrory <ma...@apache.org>
Authored: Mon Jun 25 11:04:34 2018 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Mon Jun 25 14:59:41 2018 -0600
----------------------------------------------------------------------
.../fs/s3a/s3guard/LocalMetadataEntry.java | 81 ++++++
.../fs/s3a/s3guard/LocalMetadataStore.java | 247 +++++++++++--------
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 2 +-
.../fs/s3a/s3guard/TestLocalMetadataStore.java | 33 ++-
4 files changed, 240 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c687a661/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java
new file mode 100644
index 0000000..6040d67
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import javax.annotation.Nullable;
+
+/**
+ * LocalMetadataEntry is used to store entries in the cache of
+ * LocalMetadataStore. PathMetadata or dirListingMetadata can be null. The
+ * entry is not immutable.
+ */
+public final class LocalMetadataEntry {
+ @Nullable
+ private PathMetadata pathMetadata;
+ @Nullable
+ private DirListingMetadata dirListingMetadata;
+
+ LocalMetadataEntry(PathMetadata pmd){
+ pathMetadata = pmd;
+ dirListingMetadata = null;
+ }
+
+ LocalMetadataEntry(DirListingMetadata dlm){
+ pathMetadata = null;
+ dirListingMetadata = dlm;
+ }
+
+ public PathMetadata getFileMeta() {
+ return pathMetadata;
+ }
+
+ public DirListingMetadata getDirListingMeta() {
+ return dirListingMetadata;
+ }
+
+
+ public boolean hasPathMeta() {
+ return this.pathMetadata != null;
+ }
+
+ public boolean hasDirMeta() {
+ return this.dirListingMetadata != null;
+ }
+
+ public void setPathMetadata(PathMetadata pathMetadata) {
+ this.pathMetadata = pathMetadata;
+ }
+
+ public void setDirListingMetadata(DirListingMetadata dirListingMetadata) {
+ this.dirListingMetadata = dirListingMetadata;
+ }
+
+ @Override public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("LocalMetadataEntry{");
+ if(pathMetadata != null) {
+ sb.append("pathMetadata=" + pathMetadata.getFileStatus().getPath());
+ }
+ if(dirListingMetadata != null){
+ sb.append("; dirListingMetadata=" + dirListingMetadata.getPath());
+ }
+ sb.append("}");
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c687a661/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
index 95689e1..49981ed 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
@@ -37,13 +37,12 @@ import java.io.IOException;
import java.net.URI;
import java.util.Collection;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
- * This is a local, in-memory, implementation of MetadataStore.
+ * This is a local, in-memory implementation of MetadataStore.
* This is <i>not</i> a coherent cache across processes. It is only
* locally-coherent.
*
@@ -56,12 +55,12 @@ import java.util.concurrent.TimeUnit;
* non-recursive removal of non-empty directories. It is assumed the caller
* already has to perform these sorts of checks.
*
- * Contains cache internally with time based eviction.
+ * Contains one cache internally with time based eviction.
*/
public class LocalMetadataStore implements MetadataStore {
public static final Logger LOG = LoggerFactory.getLogger(MetadataStore.class);
- public static final int DEFAULT_MAX_RECORDS = 128;
+ public static final int DEFAULT_MAX_RECORDS = 256;
public static final int DEFAULT_CACHE_ENTRY_TTL_MSEC = 10 * 1000;
/**
@@ -79,11 +78,8 @@ public class LocalMetadataStore implements MetadataStore {
public static final String CONF_CACHE_ENTRY_TTL =
"fs.metadatastore.local.ttl";
- /** Contains directories and files. */
- private Cache<Path, PathMetadata> fileCache;
-
- /** Contains directory listings. */
- private Cache<Path, DirListingMetadata> dirCache;
+ /** Contains directory and file listings. */
+ private Cache<Path, LocalMetadataEntry> localCache;
private FileSystem fs;
/* Null iff this FS does not have an associated URI host. */
@@ -116,8 +112,7 @@ public class LocalMetadataStore implements MetadataStore {
builder.expireAfterAccess(ttl, TimeUnit.MILLISECONDS);
}
- fileCache = builder.build();
- dirCache = builder.build();
+ localCache = builder.build();
}
@Override
@@ -155,8 +150,7 @@ public class LocalMetadataStore implements MetadataStore {
if (recursive) {
// Remove all entries that have this dir as path prefix.
- deleteEntryByAncestor(path, dirCache, tombstone);
- deleteEntryByAncestor(path, fileCache, tombstone);
+ deleteEntryByAncestor(path, localCache, tombstone);
}
}
@@ -170,7 +164,7 @@ public class LocalMetadataStore implements MetadataStore {
throws IOException {
Path path = standardize(p);
synchronized (this) {
- PathMetadata m = fileCache.getIfPresent(path);
+ PathMetadata m = getFileMeta(path);
if (wantEmptyDirectoryFlag && m != null &&
m.getFileStatus().isDirectory()) {
@@ -191,15 +185,15 @@ public class LocalMetadataStore implements MetadataStore {
* @return TRUE / FALSE if known empty / not-empty, UNKNOWN otherwise.
*/
private Tristate isEmptyDirectory(Path p) {
- DirListingMetadata dirMeta = dirCache.getIfPresent(p);
- return dirMeta.withoutTombstones().isEmpty();
+ DirListingMetadata dlm = getDirListingMeta(p);
+ return dlm.withoutTombstones().isEmpty();
}
@Override
public synchronized DirListingMetadata listChildren(Path p) throws
IOException {
Path path = standardize(p);
- DirListingMetadata listing = dirCache.getIfPresent(path);
+ DirListingMetadata listing = getDirListingMeta(path);
if (LOG.isDebugEnabled()) {
LOG.debug("listChildren({}) -> {}", path,
listing == null ? "null" : listing.prettyPrint());
@@ -211,6 +205,7 @@ public class LocalMetadataStore implements MetadataStore {
@Override
public void move(Collection<Path> pathsToDelete,
Collection<PathMetadata> pathsToCreate) throws IOException {
+ LOG.info("Move {} to {}", pathsToDelete, pathsToCreate);
Preconditions.checkNotNull(pathsToDelete, "pathsToDelete is null");
Preconditions.checkNotNull(pathsToCreate, "pathsToCreate is null");
@@ -258,7 +253,12 @@ public class LocalMetadataStore implements MetadataStore {
if (LOG.isDebugEnabled()) {
LOG.debug("put {} -> {}", path, meta.prettyPrint());
}
- fileCache.put(path, meta);
+ LocalMetadataEntry entry = localCache.getIfPresent(path);
+ if(entry == null){
+ entry = new LocalMetadataEntry(meta);
+ } else {
+ entry.setPathMetadata(meta);
+ }
/* Directory case:
* We also make sure we have an entry in the dirCache, so subsequent
@@ -271,27 +271,32 @@ public class LocalMetadataStore implements MetadataStore {
* saving round trips to underlying store for subsequent listStatus()
*/
- if (status.isDirectory()) {
- DirListingMetadata dir = dirCache.getIfPresent(path);
- if (dir == null) {
- dirCache.put(path, new DirListingMetadata(path, DirListingMetadata
- .EMPTY_DIR, false));
- }
+ // only create DirListingMetadata if the entry does not have one
+ if (status.isDirectory() && !entry.hasDirMeta()) {
+ DirListingMetadata dlm =
+ new DirListingMetadata(path, DirListingMetadata.EMPTY_DIR, false);
+ entry.setDirListingMetadata(dlm);
}
+ localCache.put(path, entry);
/* Update cached parent dir. */
Path parentPath = path.getParent();
if (parentPath != null) {
- DirListingMetadata parent = dirCache.getIfPresent(parentPath);
- if (parent == null) {
- /* Track this new file's listing in parent. Parent is not
- * authoritative, since there may be other items in it we don't know
- * about. */
- parent = new DirListingMetadata(parentPath,
- DirListingMetadata.EMPTY_DIR, false);
- dirCache.put(parentPath, parent);
+ LocalMetadataEntry parentMeta = localCache.getIfPresent(parentPath);
+ DirListingMetadata parentDirMeta =
+ new DirListingMetadata(parentPath, DirListingMetadata.EMPTY_DIR,
+ false);
+ parentDirMeta.put(status);
+
+ getDirListingMeta(parentPath);
+
+ if (parentMeta == null){
+ localCache.put(parentPath, new LocalMetadataEntry(parentDirMeta));
+ } else if (!parentMeta.hasDirMeta()) {
+ parentMeta.setDirListingMetadata(parentDirMeta);
+ } else {
+ parentMeta.getDirListingMeta().put(status);
}
- parent.put(status);
}
}
}
@@ -301,7 +306,13 @@ public class LocalMetadataStore implements MetadataStore {
if (LOG.isDebugEnabled()) {
LOG.debug("put dirMeta {}", meta.prettyPrint());
}
- dirCache.put(standardize(meta.getPath()), meta);
+ LocalMetadataEntry entry =
+ localCache.getIfPresent(standardize(meta.getPath()));
+ if(entry == null){
+ localCache.put(standardize(meta.getPath()), new LocalMetadataEntry(meta));
+ } else {
+ entry.setDirListingMetadata(meta);
+ }
put(meta.getListing());
}
@@ -319,8 +330,8 @@ public class LocalMetadataStore implements MetadataStore {
@Override
public void destroy() throws IOException {
- if (dirCache != null) {
- dirCache.invalidateAll();
+ if (localCache != null) {
+ localCache.invalidateAll();
}
}
@@ -330,42 +341,44 @@ public class LocalMetadataStore implements MetadataStore {
}
@Override
- public synchronized void prune(long modTime, String keyPrefix)
- throws IOException {
- Iterator<Map.Entry<Path, PathMetadata>> files =
- fileCache.asMap().entrySet().iterator();
- while (files.hasNext()) {
- Map.Entry<Path, PathMetadata> entry = files.next();
- if (expired(entry.getValue().getFileStatus(), modTime, keyPrefix)) {
- files.remove();
- }
- }
- Iterator<Map.Entry<Path, DirListingMetadata>> dirs =
- dirCache.asMap().entrySet().iterator();
- while (dirs.hasNext()) {
- Map.Entry<Path, DirListingMetadata> entry = dirs.next();
- Path path = entry.getKey();
- DirListingMetadata metadata = entry.getValue();
- Collection<PathMetadata> oldChildren = metadata.getListing();
- Collection<PathMetadata> newChildren = new LinkedList<>();
-
- for (PathMetadata child : oldChildren) {
- FileStatus status = child.getFileStatus();
- if (!expired(status, modTime, keyPrefix)) {
- newChildren.add(child);
- }
- }
- if (newChildren.size() != oldChildren.size()) {
- dirCache.put(path, new DirListingMetadata(path, newChildren, false));
- if (!path.isRoot()) {
- DirListingMetadata parent = null;
- parent = dirCache.getIfPresent(path.getParent());
- if (parent != null) {
- parent.setAuthoritative(false);
+ public synchronized void prune(long modTime, String keyPrefix) {
+ // prune files
+ // filter path_metadata (files), filter expired, remove expired
+ localCache.asMap().entrySet().stream()
+ .filter(entry -> entry.getValue().hasPathMeta())
+ .filter(entry -> expired(
+ entry.getValue().getFileMeta().getFileStatus(), modTime, keyPrefix))
+ .forEach(entry -> localCache.invalidate(entry.getKey()));
+
+
+ // prune dirs
+ // filter DIR_LISTING_METADATA, remove expired, remove authoritative bit
+ localCache.asMap().entrySet().stream()
+ .filter(entry -> entry.getValue().hasDirMeta())
+ .forEach(entry -> {
+ Path path = entry.getKey();
+ DirListingMetadata metadata = entry.getValue().getDirListingMeta();
+ Collection<PathMetadata> oldChildren = metadata.getListing();
+ Collection<PathMetadata> newChildren = new LinkedList<>();
+
+ for (PathMetadata child : oldChildren) {
+ FileStatus status = child.getFileStatus();
+ if (!expired(status, modTime, keyPrefix)) {
+ newChildren.add(child);
+ }
}
- }
- }
- }
+ if (newChildren.size() != oldChildren.size()) {
+ DirListingMetadata dlm =
+ new DirListingMetadata(path, newChildren, false);
+ localCache.put(path, new LocalMetadataEntry(dlm));
+ if (!path.isRoot()) {
+ DirListingMetadata parent = getDirListingMeta(path.getParent());
+ if (parent != null) {
+ parent.setAuthoritative(false);
+ }
+ }
+ }
+ });
}
private boolean expired(FileStatus status, long expiry, String keyPrefix) {
@@ -390,31 +403,26 @@ public class LocalMetadataStore implements MetadataStore {
}
@VisibleForTesting
- static <T> void deleteEntryByAncestor(Path ancestor, Cache<Path, T> cache,
- boolean tombstone) {
- for (Iterator<Map.Entry<Path, T>> it = cache.asMap().entrySet().iterator();
- it.hasNext();) {
- Map.Entry<Path, T> entry = it.next();
- Path f = entry.getKey();
- T meta = entry.getValue();
- if (isAncestorOf(ancestor, f)) {
- if (tombstone) {
- if (meta instanceof PathMetadata) {
- cache.put(f, (T) PathMetadata.tombstone(f));
- } else if (meta instanceof DirListingMetadata) {
- it.remove();
+ static void deleteEntryByAncestor(Path ancestor,
+ Cache<Path, LocalMetadataEntry> cache, boolean tombstone) {
+
+ cache.asMap().entrySet().stream()
+ .filter(entry -> isAncestorOf(ancestor, entry.getKey()))
+ .forEach(entry -> {
+ LocalMetadataEntry meta = entry.getValue();
+ Path path = entry.getKey();
+ if(meta.hasDirMeta()){
+ cache.invalidate(path);
+ } else if(tombstone && meta.hasPathMeta()){
+ meta.setPathMetadata(PathMetadata.tombstone(path));
} else {
- throw new IllegalStateException("Unknown type in cache");
+ cache.invalidate(path);
}
- } else {
- it.remove();
- }
- }
- }
+ });
}
/**
- * @return true iff 'ancestor' is ancestor dir in path 'f'.
+ * @return true if 'ancestor' is ancestor dir in path 'f'.
* All paths here are absolute. Dir does not count as its own ancestor.
*/
private static boolean isAncestorOf(Path ancestor, Path f) {
@@ -431,27 +439,41 @@ public class LocalMetadataStore implements MetadataStore {
* lock held.
*/
private void deleteCacheEntries(Path path, boolean tombstone) {
+ LocalMetadataEntry entry = localCache.getIfPresent(path);
+ // If there's no entry, delete should silently succeed
+ // (based on MetadataStoreTestBase#testDeleteNonExisting)
+ if(entry == null){
+ LOG.warn("Delete: path {} is missing from cache.", path);
+ return;
+ }
- // Remove target file/dir
+ // Remove target file entry
LOG.debug("delete file entry for {}", path);
- if (tombstone) {
- fileCache.put(path, PathMetadata.tombstone(path));
- } else {
- fileCache.invalidate(path);
+ if(entry.hasPathMeta()){
+ if (tombstone) {
+ PathMetadata pmd = PathMetadata.tombstone(path);
+ entry.setPathMetadata(pmd);
+ } else {
+ entry.setPathMetadata(null);
+ }
}
- // Update this and parent dir listing, if any
-
- /* If this path is a dir, remove its listing */
- LOG.debug("removing listing of {}", path);
+ // If this path is a dir, remove its listing
+ if(entry.hasDirMeta()) {
+ LOG.debug("removing listing of {}", path);
+ entry.setDirListingMetadata(null);
+ }
- dirCache.invalidate(path);
+ // If the entry is empty (contains no dirMeta or pathMeta) remove it from
+ // the cache.
+ if(!entry.hasDirMeta() && !entry.hasPathMeta()){
+ localCache.invalidate(entry);
+ }
/* Remove this path from parent's dir listing */
Path parent = path.getParent();
if (parent != null) {
- DirListingMetadata dir = null;
- dir = dirCache.getIfPresent(parent);
+ DirListingMetadata dir = getDirListingMeta(parent);
if (dir != null) {
LOG.debug("removing parent's entry for {} ", path);
if (tombstone) {
@@ -494,4 +516,23 @@ public class LocalMetadataStore implements MetadataStore {
public void updateParameters(Map<String, String> parameters)
throws IOException {
}
+
+ PathMetadata getFileMeta(Path p){
+ LocalMetadataEntry entry = localCache.getIfPresent(p);
+ if(entry != null && entry.hasPathMeta()){
+ return entry.getFileMeta();
+ } else {
+ return null;
+ }
+ }
+
+ DirListingMetadata getDirListingMeta(Path p){
+ LocalMetadataEntry entry = localCache.getIfPresent(p);
+ if(entry != null && entry.hasDirMeta()){
+ return entry.getDirListingMeta();
+ } else {
+ return null;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c687a661/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
index 56618cb..5a59400 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -836,7 +836,7 @@ public abstract class MetadataStoreTestBase extends HadoopTestBase {
throws IOException {
DirListingMetadata dirMeta = ms.listChildren(strToPath(pathStr));
if (!allowMissing()) {
- assertNotNull("Directory " + pathStr + " in cache", dirMeta);
+ assertNotNull("Directory " + pathStr + " is null in cache", dirMeta);
}
if (!allowMissing() || dirMeta != null) {
dirMeta = dirMeta.withoutTombstones();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c687a661/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
index 074319f..2ea20b2 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.fs.s3a.S3ATestUtils;
*/
public class TestLocalMetadataStore extends MetadataStoreTestBase {
- private static final String MAX_ENTRIES_STR = "16";
private final static class LocalMSContract extends AbstractMSContract {
@@ -48,7 +47,6 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
}
private LocalMSContract(Configuration config) throws IOException {
- config.set(LocalMetadataStore.CONF_MAX_RECORDS, MAX_ENTRIES_STR);
fs = FileSystem.getLocal(config);
}
@@ -76,8 +74,8 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
}
@Test
- public void testClearByAncestor() {
- Cache<Path, PathMetadata> cache = CacheBuilder.newBuilder().build();
+ public void testClearByAncestor() throws Exception {
+ Cache<Path, LocalMetadataEntry> cache = CacheBuilder.newBuilder().build();
// 1. Test paths without scheme/host
assertClearResult(cache, "", "/", 0);
@@ -122,7 +120,7 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
final long ttl = t1 + 50; // between t1 and t2
- Cache<Path, PathMetadata> cache = CacheBuilder.newBuilder()
+ Cache<Path, LocalMetadataEntry> cache = CacheBuilder.newBuilder()
.expireAfterWrite(ttl,
TimeUnit.NANOSECONDS /* nanos to avoid conversions */)
.ticker(testTicker)
@@ -143,7 +141,7 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
assertEquals("Cache should contain 3 records before eviction",
3, cache.size());
- PathMetadata pm1 = cache.getIfPresent(path1);
+ LocalMetadataEntry pm1 = cache.getIfPresent(path1);
assertNotNull("PathMetadata should not be null before eviction", pm1);
// set the ticker to a time when timed eviction should occur
@@ -159,7 +157,7 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
assertNull("PathMetadata should be null after eviction", pm1);
}
- private static void populateMap(Cache<Path, PathMetadata> cache,
+ private static void populateMap(Cache<Path, LocalMetadataEntry> cache,
String prefix) {
populateEntry(cache, new Path(prefix + "/dirA/dirB/"));
populateEntry(cache, new Path(prefix + "/dirA/dirB/dirC"));
@@ -168,23 +166,20 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
populateEntry(cache, new Path(prefix + "/dirA/file1"));
}
- private static void populateEntry(Cache<Path, PathMetadata> cache,
+ private static void populateEntry(Cache<Path, LocalMetadataEntry> cache,
Path path) {
- cache.put(path, new PathMetadata(new FileStatus(0, true, 0, 0, 0, path)));
+ FileStatus fileStatus = new FileStatus(0, true, 0, 0, 0, path);
+ cache.put(path, new LocalMetadataEntry(new PathMetadata(fileStatus)));
}
- private static int sizeOfMap(Cache<Path, PathMetadata> cache) {
- int count = 0;
- for (PathMetadata meta : cache.asMap().values()) {
- if (!meta.isDeleted()) {
- count++;
- }
- }
- return count;
+ private static long sizeOfMap(Cache<Path, LocalMetadataEntry> cache) {
+ return cache.asMap().values().stream()
+ .filter(entry -> !entry.getFileMeta().isDeleted())
+ .count();
}
- private static void assertClearResult(Cache<Path, PathMetadata> cache,
- String prefixStr, String pathStr, int leftoverSize) {
+ private static void assertClearResult(Cache<Path, LocalMetadataEntry> cache,
+ String prefixStr, String pathStr, int leftoverSize) throws IOException {
populateMap(cache, prefixStr);
LocalMetadataStore.deleteEntryByAncestor(new Path(prefixStr + pathStr),
cache, true);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: MAPREDUCE-7063. Fix log level
inconsistency in CombineFileInputFormat.java
Posted by xk...@apache.org.
MAPREDUCE-7063. Fix log level inconsistency in CombineFileInputFormat.java
Signed-off-by: Akira Ajisaka <aa...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e94e597
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e94e597
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e94e597
Branch: refs/heads/HDFS-12943
Commit: 1e94e5977f9075af2f74e30a3b8e52f7ded67863
Parents: 2c87ec5
Author: Vidura Mudalige <vi...@kohls.com>
Authored: Thu Jun 14 21:23:01 2018 +0530
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Jun 18 14:25:11 2018 -0700
----------------------------------------------------------------------
.../hadoop/mapreduce/lib/input/CombineFileInputFormat.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e94e597/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
index c7a737c..b16e127 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
@@ -425,8 +425,8 @@ public abstract class CombineFileInputFormat<K, V>
if (completedNodes.size() == totalNodes || totalLength == 0) {
// All nodes have been walked over and marked as completed or all blocks
// have been assigned. The rest should be handled via rackLock assignment.
- LOG.info("DEBUG: Terminated node allocation with : CompletedNodes: "
- + completedNodes.size() + ", size left: " + totalLength);
+ LOG.debug("Terminated node allocation with : CompletedNodes: {}, size left: {}",
+ completedNodes.size(), totalLength);
break;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HADOOP-15550. Avoid static
initialization of ObjectMappers
Posted by xk...@apache.org.
HADOOP-15550. Avoid static initialization of ObjectMappers
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a3c6e9c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a3c6e9c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a3c6e9c
Branch: refs/heads/HDFS-12943
Commit: 7a3c6e9c3cd9ffdc71946fd12f5c3d59718c4939
Parents: c687a66
Author: Todd Lipcon <to...@apache.org>
Authored: Mon Jun 25 15:36:45 2018 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Mon Jun 25 15:36:45 2018 -0700
----------------------------------------------------------------------
.../crypto/key/kms/KMSClientProvider.java | 7 ++----
.../web/DelegationTokenAuthenticator.java | 8 ++-----
.../apache/hadoop/util/HttpExceptionUtils.java | 12 ++--------
.../apache/hadoop/util/JsonSerialization.java | 24 ++++++++++++++++++++
.../crypto/key/kms/server/KMSJSONWriter.java | 6 ++---
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 7 ++----
...onfRefreshTokenBasedAccessTokenProvider.java | 8 +++----
.../CredentialBasedAccessTokenProvider.java | 8 +++----
.../apache/hadoop/mapreduce/JobSubmitter.java | 8 +++----
.../hadoop/fs/azure/security/JsonUtils.java | 4 ++--
10 files changed, 45 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index edbf897..7b46075 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.security.token.TokenRenewer;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
import org.apache.hadoop.util.HttpExceptionUtils;
+import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.KMSUtil;
import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger;
@@ -79,7 +80,6 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -132,9 +132,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
private final ValueQueue<EncryptedKeyVersion> encKeyVersionQueue;
- private static final ObjectWriter WRITER =
- new ObjectMapper().writerWithDefaultPrettyPrinter();
-
private final Text dtService;
// Allow fallback to default kms server port 9600 for certain tests that do
@@ -237,7 +234,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
private static void writeJson(Object obj, OutputStream os)
throws IOException {
Writer writer = new OutputStreamWriter(os, StandardCharsets.UTF_8);
- WRITER.writeValue(writer, obj);
+ JsonSerialization.writer().writeValue(writer, obj);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
index 617773b..0ae2af3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.security.token.delegation.web;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.net.NetUtils;
@@ -31,6 +29,7 @@ import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.util.HttpExceptionUtils;
+import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,9 +55,6 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
private static final String CONTENT_TYPE = "Content-Type";
private static final String APPLICATION_JSON_MIME = "application/json";
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(Map.class);
-
private static final String HTTP_GET = "GET";
private static final String HTTP_PUT = "PUT";
@@ -328,7 +324,7 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
if (contentType != null &&
contentType.contains(APPLICATION_JSON_MIME)) {
try {
- ret = READER.readValue(conn.getInputStream());
+ ret = JsonSerialization.mapReader().readValue(conn.getInputStream());
} catch (Exception ex) {
throw new AuthenticationException(String.format(
"'%s' did not handle the '%s' delegation token operation: %s",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
index cdb8112..50be1c3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
@@ -17,9 +17,6 @@
*/
package org.apache.hadoop.util;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -56,11 +53,6 @@ public class HttpExceptionUtils {
private static final String ENTER = System.getProperty("line.separator");
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(Map.class);
- private static final ObjectWriter WRITER =
- new ObjectMapper().writerWithDefaultPrettyPrinter();
-
/**
* Creates a HTTP servlet response serializing the exception in it as JSON.
*
@@ -82,7 +74,7 @@ public class HttpExceptionUtils {
Map<String, Object> jsonResponse = new LinkedHashMap<String, Object>();
jsonResponse.put(ERROR_JSON, json);
Writer writer = response.getWriter();
- WRITER.writeValue(writer, jsonResponse);
+ JsonSerialization.writer().writeValue(writer, jsonResponse);
writer.flush();
}
@@ -150,7 +142,7 @@ public class HttpExceptionUtils {
InputStream es = null;
try {
es = conn.getErrorStream();
- Map json = READER.readValue(es);
+ Map json = JsonSerialization.mapReader().readValue(es);
json = (Map) json.get(ERROR_JSON);
String exClass = (String) json.get(ERROR_CLASSNAME_JSON);
String exMsg = (String) json.get(ERROR_MESSAGE_JSON);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
index 86c4df6..cbc8560 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
@@ -25,14 +25,18 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.Map;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.google.common.base.Preconditions;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,6 +69,26 @@ public class JsonSerialization<T> {
private final Class<T> classType;
private final ObjectMapper mapper;
+ private static final ObjectWriter WRITER =
+ new ObjectMapper().writerWithDefaultPrettyPrinter();
+
+ private static final ObjectReader MAP_READER =
+ new ObjectMapper().readerFor(Map.class);
+
+ /**
+ * @return an ObjectWriter which pretty-prints its output
+ */
+ public static ObjectWriter writer() {
+ return WRITER;
+ }
+
+ /**
+ * @return an ObjectReader which returns simple Maps.
+ */
+ public static ObjectReader mapReader() {
+ return MAP_READER;
+ }
+
/**
* Create an instance bound to a specific type.
* @param classType class to marshall
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
index f826572..b9b8d9c 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
@@ -17,10 +17,9 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.util.JsonSerialization;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
@@ -67,8 +66,7 @@ public class KMSJSONWriter implements MessageBodyWriter<Object> {
OutputStream outputStream) throws IOException, WebApplicationException {
Writer writer = new OutputStreamWriter(outputStream, Charset
.forName("UTF-8"));
- ObjectMapper jsonMapper = new ObjectMapper();
- jsonMapper.writerWithDefaultPrettyPrinter().writeValue(writer, obj);
+ JsonSerialization.writer().writeValue(writer, obj);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 673acd6..ec60a18 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -56,8 +56,6 @@ import java.util.concurrent.TimeUnit;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.BoundedInputStream;
import org.apache.hadoop.conf.Configuration;
@@ -121,6 +119,7 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.TokenSelector;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
+import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.KMSUtil;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.StringUtils;
@@ -172,8 +171,6 @@ public class WebHdfsFileSystem extends FileSystem
private boolean disallowFallbackToInsecureCluster;
private String restCsrfCustomHeader;
private Set<String> restCsrfMethodsToIgnore;
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(Map.class);
private DFSOpsCountStatistics storageStatistics;
@@ -476,7 +473,7 @@ public class WebHdfsFileSystem extends FileSystem
+ "\" (parsed=\"" + parsed + "\")");
}
}
- return READER.readValue(in);
+ return JsonSerialization.mapReader().readValue(in);
} finally {
in.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
index c6ebdd6..3e3fbfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
@@ -18,8 +18,6 @@
*/
package org.apache.hadoop.hdfs.web.oauth2;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
import com.squareup.okhttp.OkHttpClient;
import com.squareup.okhttp.Request;
import com.squareup.okhttp.RequestBody;
@@ -28,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.Timer;
import org.apache.http.HttpStatus;
@@ -55,8 +54,6 @@ import static org.apache.hadoop.hdfs.web.oauth2.Utils.notNull;
@InterfaceStability.Evolving
public class ConfRefreshTokenBasedAccessTokenProvider
extends AccessTokenProvider {
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(Map.class);
public static final String OAUTH_REFRESH_TOKEN_KEY
= "dfs.webhdfs.oauth2.refresh.token";
@@ -129,7 +126,8 @@ public class ConfRefreshTokenBasedAccessTokenProvider
+ responseBody.code() + ", text = " + responseBody.toString());
}
- Map<?, ?> response = READER.readValue(responseBody.body().string());
+ Map<?, ?> response = JsonSerialization.mapReader().readValue(
+ responseBody.body().string());
String newExpiresIn = response.get(EXPIRES_IN).toString();
accessTokenTimer.setExpiresIn(newExpiresIn);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
index 5c629e0..bfd7055 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
@@ -18,8 +18,6 @@
*/
package org.apache.hadoop.hdfs.web.oauth2;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
import com.squareup.okhttp.OkHttpClient;
import com.squareup.okhttp.Request;
import com.squareup.okhttp.RequestBody;
@@ -28,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.Timer;
import org.apache.http.HttpStatus;
@@ -55,8 +54,6 @@ import static org.apache.hadoop.hdfs.web.oauth2.Utils.notNull;
@InterfaceStability.Evolving
public abstract class CredentialBasedAccessTokenProvider
extends AccessTokenProvider {
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(Map.class);
public static final String OAUTH_CREDENTIAL_KEY
= "dfs.webhdfs.oauth2.credential";
@@ -123,7 +120,8 @@ public abstract class CredentialBasedAccessTokenProvider
+ responseBody.code() + ", text = " + responseBody.toString());
}
- Map<?, ?> response = READER.readValue(responseBody.body().string());
+ Map<?, ?> response = JsonSerialization.mapReader().readValue(
+ responseBody.body().string());
String newExpiresIn = response.get(EXPIRES_IN).toString();
timer.setExpiresIn(newExpiresIn);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
index 246986f..ebf9341 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
@@ -34,8 +34,6 @@ import javax.crypto.SecretKey;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -61,6 +59,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.api.records.ReservationId;
@@ -71,8 +70,6 @@ import com.google.common.base.Charsets;
class JobSubmitter {
protected static final Logger LOG =
LoggerFactory.getLogger(JobSubmitter.class);
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(Map.class);
private static final String SHUFFLE_KEYGEN_ALGORITHM = "HmacSHA1";
private static final int SHUFFLE_KEY_LENGTH = 64;
private FileSystem jtFs;
@@ -406,7 +403,8 @@ class JobSubmitter {
try {
// read JSON
- Map<String, String> nm = READER.readValue(new File(localFileName));
+ Map<String, String> nm = JsonSerialization.mapReader().readValue(
+ new File(localFileName));
for(Map.Entry<String, String> ent: nm.entrySet()) {
credentials.addSecretKey(new Text(ent.getKey()), ent.getValue()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
index 20dd470..9c40325 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.azure.security;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.util.JsonSerialization;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,8 +38,7 @@ public final class JsonUtils {
public static Map<?, ?> parse(final String jsonString) throws IOException {
try {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readerFor(Map.class).readValue(jsonString);
+ return JsonSerialization.mapReader().readValue(jsonString);
} catch (Exception e) {
LOG.debug("JSON Parsing exception: {} while parsing {}", e.getMessage(),
jsonString);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: MAPREDUCE-7113. Typos in test
names in TestTaskAttempt: "testAppDiognostic". Contributed by Szilard Nemeth.
Posted by xk...@apache.org.
MAPREDUCE-7113. Typos in test names in TestTaskAttempt: "testAppDiognostic". Contributed by Szilard Nemeth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6ee4290
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6ee4290
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6ee4290
Branch: refs/heads/HDFS-12943
Commit: d6ee4290df10ad3e0b087c21accd602508e0a197
Parents: 46f9058
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Wed Jun 20 12:04:44 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Wed Jun 20 12:04:44 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6ee4290/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index 609923f..b1b7b8f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -853,7 +853,7 @@ public class TestTaskAttempt{
@Test
- public void testAppDiognosticEventOnUnassignedTask() throws Exception {
+ public void testAppDiagnosticEventOnUnassignedTask() {
ApplicationId appId = ApplicationId.newInstance(1, 2);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
appId, 0);
@@ -978,7 +978,7 @@ public class TestTaskAttempt{
}
@Test
- public void testAppDiognosticEventOnNewTask() throws Exception {
+ public void testAppDiagnosticEventOnNewTask() {
ApplicationId appId = ApplicationId.newInstance(1, 2);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
appId, 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HDDS-184. Upgrade common-langs
version to 3.7 in hadoop-tools/hadoop-ozone. Contributed by Takanobu Asanuma.
Posted by xk...@apache.org.
HDDS-184. Upgrade common-langs version to 3.7 in hadoop-tools/hadoop-ozone.
Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca14fec0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca14fec0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca14fec0
Branch: refs/heads/HDFS-12943
Commit: ca14fec02fb14e1b708f266bc715e84ae9784d6f
Parents: 8a32bc3
Author: Anu Engineer <ae...@apache.org>
Authored: Sat Jun 23 15:49:44 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Sat Jun 23 15:49:44 2018 -0700
----------------------------------------------------------------------
.../ozone/genconf/TestGenerateOzoneRequiredConfigurations.java | 2 +-
.../src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java | 2 +-
.../java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java | 2 +-
.../java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java | 2 +-
.../java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java | 2 +-
5 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
index cfd1159..c2f5eb7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.ozone.genconf;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.hamcrest.CoreMatchers;
import org.junit.AfterClass;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index 0ff1d50..6906a9d 100644
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++ b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.http.client.utils.URIBuilder;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
index b82c4a1..ad21f28 100644
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.ozone;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FSDataOutputStream;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
index 5a7cb4f..a225702 100644
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
+++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
@@ -31,7 +31,7 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.junit.After;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
index 176b614..8417e46 100644
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
+++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.ozone.contract;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.fs.FileSystem;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDFS-13692.
StorageInfoDefragmenter floods log when compacting StorageInfo TreeSet.
Contributed by Bharat Viswanadham.
Posted by xk...@apache.org.
HDFS-13692. StorageInfoDefragmenter floods log when compacting StorageInfo TreeSet. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30728ace
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30728ace
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30728ace
Branch: refs/heads/HDFS-12943
Commit: 30728aced4a6b05394b3fc8c613f39fade9cf3c2
Parents: 9994856
Author: Yiqun Lin <yq...@apache.org>
Authored: Fri Jun 22 10:50:54 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Fri Jun 22 10:50:54 2018 +0800
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/blockmanagement/BlockManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30728ace/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 76a7781..72ea1c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -4576,7 +4576,7 @@ public class BlockManager implements BlockStatsMXBean {
datanodesAndStorages.add(node.getDatanodeUuid());
datanodesAndStorages.add(storage.getStorageID());
}
- LOG.info("StorageInfo TreeSet fill ratio {} : {}{}",
+ LOG.debug("StorageInfo TreeSet fill ratio {} : {}{}",
storage.getStorageID(), ratio,
(ratio < storageInfoDefragmentRatio)
? " (queued for defragmentation)" : "");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HADOOP-15527. Improve delay check
for stopping processes. Contributed by Vinod Kumar Vavilapalli
Posted by xk...@apache.org.
HADOOP-15527. Improve delay check for stopping processes.
Contributed by Vinod Kumar Vavilapalli
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c87ec5a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c87ec5a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c87ec5a
Branch: refs/heads/HDFS-12943
Commit: 2c87ec5affefeb1dc794c4eaae685a4e544f1841
Parents: fba9d7c
Author: Eric Yang <ey...@apache.org>
Authored: Mon Jun 18 14:28:22 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Mon Jun 18 14:28:22 2018 -0400
----------------------------------------------------------------------
.../test/scripts/process_with_sigterm_trap.sh | 24 ++++++++++++++++++++
1 file changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c87ec5a/hadoop-common/src/test/scripts/process_with_sigterm_trap.sh
----------------------------------------------------------------------
diff --git a/hadoop-common/src/test/scripts/process_with_sigterm_trap.sh b/hadoop-common/src/test/scripts/process_with_sigterm_trap.sh
new file mode 100644
index 0000000..d7c7427
--- /dev/null
+++ b/hadoop-common/src/test/scripts/process_with_sigterm_trap.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+trap "echo SIGTERM trapped!" SIGTERM
+trap "echo SIGINT trapped!" SIGINT
+
+echo "$$" > "$1"
+
+while true; do
+ sleep 1.3
+done
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-8440. Typo in
YarnConfiguration javadoc: "Miniumum request grant-able..". Contributed by
Szilard Nemeth.
Posted by xk...@apache.org.
YARN-8440. Typo in YarnConfiguration javadoc: "Miniumum request grant-able..". Contributed by Szilard Nemeth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55432b09
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55432b09
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55432b09
Branch: refs/heads/HDFS-12943
Commit: 55432b09810b59ee361d0d4a8958efabb49fab3c
Parents: 9a9e969
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Wed Jun 20 11:36:12 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Wed Jun 20 11:36:12 2018 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55432b09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 5292a25..5842d64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -240,7 +240,7 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" +
DEFAULT_RM_SCHEDULER_PORT;
- /** Miniumum request grant-able by the RM scheduler. */
+ /** Minimum request grant-able by the RM scheduler. */
public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB =
YARN_PREFIX + "scheduler.minimum-allocation-mb";
public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-8443. Total #VCores in
cluster metrics is wrong when CapacityScheduler reserved some containers.
Contributed by Tao Yang.
Posted by xk...@apache.org.
YARN-8443. Total #VCores in cluster metrics is wrong when CapacityScheduler reserved some containers. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/440140ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/440140ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/440140ce
Branch: refs/heads/HDFS-12943
Commit: 440140cea6718229094a3d2b97b9b9bd28b95d9b
Parents: e16e5b3
Author: Weiwei Yang <ww...@apache.org>
Authored: Mon Jun 25 09:15:31 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Mon Jun 25 09:15:31 2018 +0800
----------------------------------------------------------------------
.../yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/440140ce/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 84f70d9..69d88aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -101,7 +101,7 @@ public class ClusterMetricsInfo {
CapacityScheduler cs = (CapacityScheduler) rs;
this.totalMB = availableMB + allocatedMB + reservedMB;
this.totalVirtualCores =
- availableVirtualCores + allocatedVirtualCores + containersReserved;
+ availableVirtualCores + allocatedVirtualCores + reservedVirtualCores;
// TODO, add support of other schedulers to get total used resources
// across partition.
if (cs.getRootQueue() != null
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HADOOP-15458.
TestLocalFileSystem#testFSOutputStreamBuilder fails on Windows. Contributed
by Xiao Liang.
Posted by xk...@apache.org.
HADOOP-15458. TestLocalFileSystem#testFSOutputStreamBuilder fails on Windows. Contributed by Xiao Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/abc3e4ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/abc3e4ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/abc3e4ba
Branch: refs/heads/HDFS-12943
Commit: abc3e4bad905efde5a4881e8a072c68f6e910ade
Parents: 1ba4e62
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Jun 25 09:50:27 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Mon Jun 25 09:50:27 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/TestLocalFileSystem.java | 23 ++++++++++----------
1 file changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/abc3e4ba/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
index 0e337b4..d5622af 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
@@ -689,17 +689,18 @@ public class TestLocalFileSystem {
// and permission
FSDataOutputStreamBuilder builder =
fileSys.createFile(path);
- builder.build();
- Assert.assertEquals("Should be default block size",
- builder.getBlockSize(), fileSys.getDefaultBlockSize());
- Assert.assertEquals("Should be default replication factor",
- builder.getReplication(), fileSys.getDefaultReplication());
- Assert.assertEquals("Should be default buffer size",
- builder.getBufferSize(),
- fileSys.getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
- IO_FILE_BUFFER_SIZE_DEFAULT));
- Assert.assertEquals("Should be default permission",
- builder.getPermission(), FsPermission.getFileDefault());
+ try (FSDataOutputStream stream = builder.build()) {
+ Assert.assertEquals("Should be default block size",
+ builder.getBlockSize(), fileSys.getDefaultBlockSize());
+ Assert.assertEquals("Should be default replication factor",
+ builder.getReplication(), fileSys.getDefaultReplication());
+ Assert.assertEquals("Should be default buffer size",
+ builder.getBufferSize(),
+ fileSys.getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
+ IO_FILE_BUFFER_SIZE_DEFAULT));
+ Assert.assertEquals("Should be default permission",
+ builder.getPermission(), FsPermission.getFileDefault());
+ }
// Test set 0 to replication, block size and buffer size
builder = fileSys.createFile(path);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HADOOP-14396. Add builder
interface to FileContext. Contributed by Lei (Eddy) Xu.
Posted by xk...@apache.org.
HADOOP-14396. Add builder interface to FileContext.
Contributed by Lei (Eddy) Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ba4e623
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ba4e623
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ba4e623
Branch: refs/heads/HDFS-12943
Commit: 1ba4e62304a70d53f1a4f76995b6e1fac3107922
Parents: 440140c
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 25 14:38:33 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 25 14:38:57 2018 +0100
----------------------------------------------------------------------
.../hadoop/fs/FSDataOutputStreamBuilder.java | 22 +++++++
.../java/org/apache/hadoop/fs/FileContext.java | 66 ++++++++++++++++++++
.../main/java/org/apache/hadoop/fs/Options.java | 3 +
.../fs/FileContextMainOperationsBaseTest.java | 44 ++++++++++++-
4 files changed, 134 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
index 86c284a..d431293 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
@@ -116,6 +116,27 @@ public abstract class FSDataOutputStreamBuilder
protected abstract B getThisBuilder();
/**
+ * Construct from a {@link FileContext}.
+ *
+ * @param fc FileContext
+ * @param p path.
+ * @throws IOException
+ */
+ FSDataOutputStreamBuilder(@Nonnull FileContext fc,
+ @Nonnull Path p) throws IOException {
+ Preconditions.checkNotNull(fc);
+ Preconditions.checkNotNull(p);
+ this.fs = null;
+ this.path = p;
+
+ AbstractFileSystem afs = fc.getFSofPath(p);
+ FsServerDefaults defaults = afs.getServerDefaults(p);
+ bufferSize = defaults.getFileBufferSize();
+ replication = defaults.getReplication();
+ blockSize = defaults.getBlockSize();
+ }
+
+ /**
* Constructor.
*/
protected FSDataOutputStreamBuilder(@Nonnull FileSystem fileSystem,
@@ -131,6 +152,7 @@ public abstract class FSDataOutputStreamBuilder
}
protected FileSystem getFS() {
+ Preconditions.checkNotNull(fs);
return fs;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 6ea69d0..5215c3c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -24,6 +24,7 @@ import java.io.OutputStream;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
@@ -35,6 +36,8 @@ import java.util.Stack;
import java.util.TreeSet;
import java.util.Map.Entry;
+import javax.annotation.Nonnull;
+
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -695,6 +698,69 @@ public class FileContext {
}
/**
+ * {@link FSDataOutputStreamBuilder} for {@liink FileContext}.
+ */
+ private static final class FCDataOutputStreamBuilder extends
+ FSDataOutputStreamBuilder<
+ FSDataOutputStream, FCDataOutputStreamBuilder> {
+ private final FileContext fc;
+
+ private FCDataOutputStreamBuilder(
+ @Nonnull FileContext fc, @Nonnull Path p) throws IOException {
+ super(fc, p);
+ this.fc = fc;
+ Preconditions.checkNotNull(fc);
+ }
+
+ @Override
+ protected FCDataOutputStreamBuilder getThisBuilder() {
+ return this;
+ }
+
+ @Override
+ public FSDataOutputStream build() throws IOException {
+ final EnumSet<CreateFlag> flags = getFlags();
+ List<CreateOpts> createOpts = new ArrayList<>(Arrays.asList(
+ CreateOpts.blockSize(getBlockSize()),
+ CreateOpts.bufferSize(getBufferSize()),
+ CreateOpts.repFac(getReplication()),
+ CreateOpts.perms(getPermission())
+ ));
+ if (getChecksumOpt() != null) {
+ createOpts.add(CreateOpts.checksumParam(getChecksumOpt()));
+ }
+ if (getProgress() != null) {
+ createOpts.add(CreateOpts.progress(getProgress()));
+ }
+ if (isRecursive()) {
+ createOpts.add(CreateOpts.createParent());
+ }
+ return fc.create(getPath(), flags,
+ createOpts.toArray(new CreateOpts[0]));
+ }
+ }
+
+ /**
+ * Create a {@link FSDataOutputStreamBuilder} for creating or overwriting
+ * a file on indicated path.
+ *
+ * @param f the file path to create builder for.
+ * @return {@link FSDataOutputStreamBuilder} to build a
+ * {@link FSDataOutputStream}.
+ *
+ * Upon {@link FSDataOutputStreamBuilder#build()} being invoked,
+ * builder parameters will be verified by {@link FileContext} and
+ * {@link AbstractFileSystem#create}. And filesystem states will be modified.
+ *
+ * Client should expect {@link FSDataOutputStreamBuilder#build()} throw the
+ * same exceptions as create(Path, EnumSet, CreateOpts...).
+ */
+ public FSDataOutputStreamBuilder<FSDataOutputStream, ?> create(final Path f)
+ throws IOException {
+ return new FCDataOutputStreamBuilder(this, f).create();
+ }
+
+ /**
* Make(create) a directory and all the non-existent parents.
*
* @param dir - the dir to make
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
index 126e754..5e93286 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
@@ -55,6 +55,9 @@ public final class Options {
ChecksumOpt csumOpt) {
return new ChecksumParam(csumOpt);
}
+ public static Progress progress(Progressable prog) {
+ return new Progress(prog);
+ }
public static Perms perms(FsPermission perm) {
return new Perms(perm);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
index 35ec4ff..62ecd9f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
@@ -810,7 +810,49 @@ public abstract class FileContextMainOperationsBaseTest {
fc.create(p, EnumSet.of(CREATE, APPEND, OVERWRITE));
Assert.fail("Excepted exception not thrown");
}
-
+
+ @Test
+ public void testBuilderCreateNonExistingFile() throws IOException {
+ Path p = getTestRootPath(fc, "test/testBuilderCreateNonExistingFile");
+ FSDataOutputStream out = fc.create(p).build();
+ writeData(fc, p, out, data, data.length);
+ }
+
+ @Test
+ public void testBuilderCreateExistingFile() throws IOException {
+ Path p = getTestRootPath(fc, "test/testBuilderCreateExistingFile");
+ createFile(p);
+ FSDataOutputStream out = fc.create(p).overwrite(true).build();
+ writeData(fc, p, out, data, data.length);
+ }
+
+ @Test
+ public void testBuilderCreateAppendNonExistingFile() throws IOException {
+ Path p = getTestRootPath(fc, "test/testBuilderCreateAppendNonExistingFile");
+ FSDataOutputStream out = fc.create(p).append().build();
+ writeData(fc, p, out, data, data.length);
+ }
+
+ @Test
+ public void testBuilderCreateAppendExistingFile() throws IOException {
+ Path p = getTestRootPath(fc, "test/testBuilderCreateAppendExistingFile");
+ createFile(p);
+ FSDataOutputStream out = fc.create(p).append().build();
+ writeData(fc, p, out, data, 2 * data.length);
+ }
+
+ @Test
+ public void testBuilderCreateRecursive() throws IOException {
+ Path p = getTestRootPath(fc, "test/parent/no/exist/file1");
+ try (FSDataOutputStream out = fc.create(p).build()) {
+ fail("Should throw FileNotFoundException on non-exist directory");
+ } catch (FileNotFoundException e) {
+ }
+
+ FSDataOutputStream out = fc.create(p).recursive().build();
+ writeData(fc, p, out, data, data.length);
+ }
+
private static void writeData(FileContext fc, Path p, FSDataOutputStream out,
byte[] data, long expectedLen) throws IOException {
out.write(data, 0, data.length);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-8184. Too many metrics if
containerLocalizer/ResourceLocalizationService uses ReadWriteDiskValidator.
Contributed by Yufei Gu
Posted by xk...@apache.org.
YARN-8184. Too many metrics if containerLocalizer/ResourceLocalizationService uses ReadWriteDiskValidator. Contributed by Yufei Gu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1cdce86d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1cdce86d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1cdce86d
Branch: refs/heads/HDFS-12943
Commit: 1cdce86d33d4b73ba6dd4136c966eb7e822b6f36
Parents: ae05562
Author: Yufei Gu <yu...@apple.com>
Authored: Fri Jun 22 14:02:32 2018 -0700
Committer: Yufei Gu <yu...@apple.com>
Committed: Fri Jun 22 14:03:55 2018 -0700
----------------------------------------------------------------------
.../containermanager/localizer/ContainerLocalizer.java | 5 +----
.../containermanager/localizer/ResourceLocalizationService.java | 5 +----
2 files changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cdce86d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index 6a384ae..c034369 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -132,10 +132,7 @@ public class ContainerLocalizer {
this.recordFactory = recordFactory;
this.conf = new YarnConfiguration();
this.diskValidator = DiskValidatorFactory.getInstance(
- conf.get(YarnConfiguration.DISK_VALIDATOR,
- YarnConfiguration.DEFAULT_DISK_VALIDATOR));
- LOG.info("Disk Validator: " + YarnConfiguration.DISK_VALIDATOR +
- " is loaded.");
+ YarnConfiguration.DEFAULT_DISK_VALIDATOR);
this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
this.pendingResources = new HashMap<LocalResource,Future<Path>>();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cdce86d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index ddae2ae..3f0a6fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -264,10 +264,7 @@ public class ResourceLocalizationService extends CompositeService
}
diskValidator = DiskValidatorFactory.getInstance(
- conf.get(YarnConfiguration.DISK_VALIDATOR,
- YarnConfiguration.DEFAULT_DISK_VALIDATOR));
- LOG.info("Disk Validator: " + YarnConfiguration.DISK_VALIDATOR +
- " is loaded.");
+ YarnConfiguration.DEFAULT_DISK_VALIDATOR);
cacheTargetSize =
conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_TARGET_SIZE_MB, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB) << 20;
cacheCleanupPeriod =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: YARN-8326. Removed exit code file
check for launched container. Contributed by Shane Kumpf
Posted by xk...@apache.org.
YARN-8326. Removed exit code file check for launched container.
Contributed by Shane Kumpf
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a32bc39
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a32bc39
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a32bc39
Branch: refs/heads/HDFS-12943
Commit: 8a32bc39eb210fca8052c472601e24c2446b4cc2
Parents: 1cdce86
Author: Eric Yang <ey...@apache.org>
Authored: Fri Jun 22 19:12:48 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Fri Jun 22 19:12:48 2018 -0400
----------------------------------------------------------------------
.../launcher/ContainerLaunch.java | 19 -------------------
1 file changed, 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a32bc39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index bb842af..04295e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -808,25 +808,6 @@ public class ContainerLaunch implements Callable<Integer> {
}
}
- final int sleepMsec = 100;
- int msecLeft = 2000;
- if (pidFilePath != null) {
- File file = new File(getExitCodeFile(pidFilePath.toString()));
- while (!file.exists() && msecLeft >= 0) {
- try {
- Thread.sleep(sleepMsec);
- } catch (InterruptedException e) {
- }
- msecLeft -= sleepMsec;
- }
- if (msecLeft < 0) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Timeout while waiting for the exit code file: "
- + file.getAbsolutePath());
- }
- }
- }
-
// Reap the container
boolean result = exec.reapContainer(
new ContainerReapContext.Builder()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org