You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/06/11 17:50:35 UTC
[01/16] hadoop git commit: HADOOP-15516. Add test cases to cover
FileUtil#readLink. Contributed by Giovanni Matteo Fumarola.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-12943 990a22181 -> 9756c2cbe
HADOOP-15516. Add test cases to cover FileUtil#readLink. Contributed by Giovanni Matteo Fumarola.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12be8bad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12be8bad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12be8bad
Branch: refs/heads/HDFS-12943
Commit: 12be8bad7debd67c9ea72b979a39c8cf42c5f37d
Parents: 7969cc4
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Jun 7 13:34:52 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Jun 7 13:34:52 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileUtil.java | 6 +++
.../java/org/apache/hadoop/fs/TestFileUtil.java | 52 ++++++++++++++++++++
2 files changed, 58 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12be8bad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index ed10f1c..df89598 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -198,6 +198,12 @@ public class FileUtil {
* use getCanonicalPath in File to get the target of the symlink but that
* does not indicate if the given path refers to a symlink.
*/
+
+ if (f == null) {
+ LOG.warn("Can not read a null symLink");
+ return "";
+ }
+
try {
return Shell.execCommand(
Shell.getReadlinkCommand(f.toString())).trim();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12be8bad/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 01fa563..f557103 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -1441,4 +1441,56 @@ public class TestFileUtil {
}
}
+ /**
+ * This test validates the correctness of {@link FileUtil#readLink(File)} in
+ * case of null pointer inputs.
+ */
+ @Test
+ public void testReadSymlinkWithNullInput() {
+ String result = FileUtil.readLink(null);
+ Assert.assertEquals("", result);
+ }
+
+ /**
+ * This test validates the correctness of {@link FileUtil#readLink(File)}.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testReadSymlink() throws IOException {
+ Assert.assertFalse(del.exists());
+ del.mkdirs();
+
+ File file = new File(del, FILE);
+ File link = new File(del, "_link");
+
+ // Create a symbolic link
+ FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
+
+ String result = FileUtil.readLink(link);
+ Assert.assertEquals(file.getAbsolutePath(), result);
+
+ file.delete();
+ link.delete();
+ }
+
+ /**
+ * This test validates the correctness of {@link FileUtil#readLink(File)} when
+ * it gets a file in input.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testReadSymlinkWithAFileAsInput() throws IOException {
+ Assert.assertFalse(del.exists());
+ del.mkdirs();
+
+ File file = new File(del, FILE);
+
+ String result = FileUtil.readLink(file);
+ Assert.assertEquals("", result);
+
+ file.delete();
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/16] hadoop git commit: HDDS-147. Update Ozone site docs.
Contributed by Arpit Agarwal.
Posted by su...@apache.org.
HDDS-147. Update Ozone site docs. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba303b1f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba303b1f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba303b1f
Branch: refs/heads/HDFS-12943
Commit: ba303b1f890ccd4deb806cb030e26a77e316ebe4
Parents: 12be8ba
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Jun 7 14:10:52 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Jun 7 14:11:20 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/docs/content/CommandShell.md | 141 ++++-----
hadoop-ozone/docs/content/GettingStarted.md | 353 ++++++++++++-----------
2 files changed, 254 insertions(+), 240 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba303b1f/hadoop-ozone/docs/content/CommandShell.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/CommandShell.md b/hadoop-ozone/docs/content/CommandShell.md
index d8a733a..95820e9 100644
--- a/hadoop-ozone/docs/content/CommandShell.md
+++ b/hadoop-ozone/docs/content/CommandShell.md
@@ -15,139 +15,144 @@ menu: main
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
-Ozone Command Shell
-===================
+# Ozone Command Shell
-Ozone command shell gives a command shell interface to work against ozone.
+
+Ozone command shell gives a command shell interface to work against Ozone.
Please note that this document assumes that cluster is deployed
with simple authentication.
The Ozone commands take the following format.
+```
+ozone oz --command_ /volume/bucket/key -user <username> [-root]
+```
-* `ozone oz --command_ http://hostname:port/volume/bucket/key -user
-<name> -root`
-
-The *port* specified in command should match the port mentioned in the config
+The `port` specified in command should match the port mentioned in the config
property `hdds.rest.http-address`. This property can be set in `ozone-site.xml`.
The default value for the port is `9880` and is used in below commands.
-The *-root* option is a command line short cut that allows *ozone oz*
+The `-root` option is a command line short cut that allows *ozone oz*
commands to be run as the user that started the cluster. This is useful to
indicate that you want the commands to be run as some admin user. The only
reason for this option is that it makes the life of a lazy developer more
easier.
-Ozone Volume Commands
---------------------
+## Volume Commands
+
The volume commands allow users to create, delete and list the volumes in the
ozone cluster.
### Create Volume
-
-Volumes can be created only by Admins. Here is an example of creating a volume.
-
-* `ozone oz -createVolume http://localhost:9880/hive -user bilbo -quota
-100TB -root`
-
+Volumes can be created only by administrators. Here is an example of creating a volume.
+```
+ozone oz -createVolume hive -user bilbo -quota 100TB -root
+```
The above command creates a volume called `hive` owned by user `bilbo`. The
`-root` option allows the command to be executed as user `hdfs` which is an
admin in the cluster.
### Update Volume
-
Updates information like ownership and quota on an existing volume.
-
-* `ozone oz -updateVolume http://localhost:9880/hive -quota 500TB -root`
+```
+ozone oz -updateVolume hive -quota 500TB -root
+```
The above command changes the volume quota of hive from 100TB to 500TB.
### Delete Volume
Deletes a Volume if it is empty.
-
-* `ozone oz -deleteVolume http://localhost:9880/hive -root`
-
+```
+ozone oz -deleteVolume /hive -root
+```
### Info Volume
-Info volume command allows the owner or the administrator of the cluster to read meta-data about a specific volume.
-
-* `ozone oz -infoVolume http://localhost:9880/hive -root`
+Info volume command allows the owner or the administrator of the cluster
+to read meta-data about a specific volume.
+```
+ozone oz -infoVolume /hive -root
+```
### List Volumes
-
-List volume command can be used by administrator to list volumes of any user. It can also be used by a user to list volumes owned by him.
-
-* `ozone oz -listVolume http://localhost:9880/ -user bilbo -root`
+List volume command can be used by administrator to list volumes of any
+user. It can also be used by any user to list their own volumes.
+```
+ozone oz -listVolume / -user bilbo
+```
The above command lists all volumes owned by user bilbo.
-Ozone Bucket Commands
---------------------
-
-Bucket commands follow a similar pattern as volume commands. However bucket commands are designed to be run by the owner of the volume.
-Following examples assume that these commands are run by the owner of the volume or bucket.
+## Bucket Commands
+Bucket commands follow a similar pattern as volume commands. However bucket
+commands are designed to be run by the owner of the volume.
+Following examples assume that these commands are run by the owner of the
+volume or bucket.
### Create Bucket
-
Create bucket call allows the owner of a volume to create a bucket.
-
-* `ozone oz -createBucket http://localhost:9880/hive/january`
+```
+ozone oz -createBucket /hive/january
+```
This call creates a bucket called `january` in the volume called `hive`. If
the volume does not exist, then this call will fail.
-
### Update Bucket
Updates bucket meta-data, like ACLs.
-
-* `ozone oz -updateBucket http://localhost:9880/hive/january -addAcl
-user:spark:rw`
-
+```
+ozone oz -updateBucket /hive/january -addAcl user:spark:rw
+```
### Delete Bucket
Deletes a bucket if it is empty.
-
-* `ozone oz -deleteBucket http://localhost:9880/hive/january`
+```
+ozone oz -deleteBucket /hive/january
+```
### Info Bucket
Returns information about a given bucket.
-
-* `ozone oz -infoBucket http://localhost:9880/hive/january`
+```
+ozone oz -infoBucket /hive/january
+```
### List Buckets
-List buckets on a given volume.
-
-* `ozone oz -listBucket http://localhost:9880/hive`
+List buckets in a given volume.
+```
+ozone oz -listBucket /hive
+```
-Ozone Key Commands
-------------------
+## Ozone Key Commands
-Ozone key commands allows users to put, delete and get keys from ozone buckets.
+Ozone key commands allows users to put, delete and get keys from Ozone buckets.
### Put Key
-Creates or overwrites a key in ozone store, -file points to the file you want
+Creates or overwrites a key in Ozone store, -file points to the file you want
to upload.
-
-* `ozone oz -putKey http://localhost:9880/hive/january/processed.orc -file
-processed.orc`
+```
+ozone oz -putKey /hive/january/processed.orc -file processed.orc
+```
### Get Key
-Downloads a file from the ozone bucket.
-
-* `ozone oz -getKey http://localhost:9880/hive/january/processed.orc -file
- processed.orc.copy`
+Downloads a file from the Ozone bucket.
+```
+ozone oz -getKey /hive/january/processed.orc -file processed.orc.copy
+```
### Delete Key
-Deletes a key from the ozone store.
-
-* `ozone oz -deleteKey http://localhost:9880/hive/january/processed.orc`
+Deletes a key from the Ozone store.
+```
+ozone oz -deleteKey /hive/january/processed.orc
+```
### Info Key
-Reads key metadata from the ozone store.
-
-* `ozone oz -infoKey http://localhost:9880/hive/january/processed.orc`
+Reads key metadata from the Ozone store.
+```
+ozone oz -infoKey /hive/january/processed.orc
+```
### List Keys
-List all keys in an ozone bucket.
+List all keys in an Ozone bucket.
+```
+ozone oz -listKey /hive/january
+```
-* `ozone oz -listKey http://localhost:9880/hive/january`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba303b1f/hadoop-ozone/docs/content/GettingStarted.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/GettingStarted.md b/hadoop-ozone/docs/content/GettingStarted.md
index 6b2316e..531d192 100644
--- a/hadoop-ozone/docs/content/GettingStarted.md
+++ b/hadoop-ozone/docs/content/GettingStarted.md
@@ -17,118 +17,144 @@ menu: main
limitations under the License. See accompanying LICENSE file.
-->
-Ozone - Object store for Hadoop
-==============================
-
-Introduction
-------------
-Ozone is an object store for Hadoop. It is a redundant, distributed object
-store build by leveraging primitives present in HDFS. Ozone supports REST
-API for accessing the store.
-
-Getting Started
----------------
-Ozone is a work in progress and currently lives in the hadoop source tree.
-The subprojects (ozone/hdds) are part of the hadoop source tree but by default
-not compiled and not part of the official releases. To
-use it, you have to build a package by yourself and deploy a cluster.
+# Ozone - Object store for Apache Hadoop
-### Building Ozone
-To build Ozone, please checkout the hadoop sources from github. Then
-checkout the trunk branch and build it.
+## Introduction
-`mvn clean package -DskipTests=true -Dmaven.javadoc.skip=true -Pdist -Phdds -Dtar -DskipShade`
+Ozone is a scalable distributed object store for Hadoop. Ozone supports RPC
+and REST APIs for working with Volumes, Buckets and Keys.
-skipShade is just to make compilation faster and not really required.
+Existing Hadoop applications can use Ozone transparently via a Hadoop Compatible
+FileSystem shim.
-This will give you a tarball in your distribution directory. This is the
-tarball that can be used for deploying your hadoop cluster. Here is an
-example of the tarball that will be generated.
+### Basic terminology
+1. **Volumes** - Volumes are a notion similar to accounts. Volumes can be
+created or deleted only by administrators.
+1. **Buckets** - A volume can contain zero or more buckets.
+1. **Keys** - Keys are unique within a given bucket.
-* `~/apache/hadoop/hadoop-dist/target/${project.version}.tar.gz`
+### Services in a minimal Ozone cluster
+1. **Ozone Manager (OM)** - stores Ozone Metadata namely Volumes,
+Buckets and Key names.
+1. **Storage Container Manager (SCM)** - handles Storage Container lifecycle.
+Containers are the unit of replication in Ozone and not exposed to users.
+1. **DataNodes** - These are HDFS DataNodes which understand how to store
+Ozone Containers. Ozone has been designed to efficiently share storage space
+with HDFS blocks.
-At this point we have an option to setup a physical cluster or run ozone via
-docker.
+## Getting Started
-Running Ozone via Docker
-------------------------
+Ozone is currently work-in-progress and lives in the Hadoop source tree.
+The sub-projects (`hadoop-ozone` and `hadoop-hdds`) are part of
+the Hadoop source tree but they are not compiled by default and not
+part of official Apache Hadoop releases.
-This assumes that you have a running docker setup on the machine. Please run
-these following commands to see ozone in action.
+To use Ozone, you have to build a package by yourself and deploy a cluster.
- Go to the directory where the docker compose files exist.
+### Building Ozone
+To build Ozone, please checkout the Hadoop sources from the
+[Apache Hadoop git repo](https://git-wip-us.apache.org/repos/asf?p=hadoop.git).
+Then checkout the `trunk` branch and build it with the `hdds` profile enabled.
- - `cd hadoop-dist/target/compose/ozone`
+`
+git checkout trunk
+mvn clean package -DskipTests=true -Dmaven.javadoc.skip=true -Pdist -Phdds -Dtar -DskipShade
+`
-Tell docker to start ozone, this will start a KSM, SCM and a single datanode in
-the background.
+`skipShade` is just to make compilation faster and not required.
+This builds a tarball in your distribution directory which can be used to deploy your
+Ozone cluster. The tarball path is `hadoop-dist/target/ozone-${project.version}.tar.gz`.
- - `docker-compose up -d`
+At this point you can either setup a physical cluster or run Ozone via
+docker.
-Now let us run some work load against ozone, to do that we will run freon.
+### Running Ozone via Docker
-This will log into the datanode and run bash.
+This is the quickest way to bring up an Ozone cluster for development/testing
+or if you just want to get a feel for Ozone. It assumes that you have docker installed
+on the machine.
- - `docker-compose exec datanode bash`
+Go to the directory where the docker compose files exist and tell
+`docker-compose` to start Ozone. This will start SCM, OM and a single datanode
+in the background.
+```
+cd hadoop-dist/target/compose/ozone
-Now you can run the `ozone` command shell or freon, the ozone load generator.
+docker-compose up -d
+```
-This is the command to run freon.
+Now let us run some workload against Ozone. To do that we will run
+_freon_, the Ozone load generator after logging into one of the docker
+containers for OM, SCM or DataNode. Let's take DataNode for example:.
+```
+docker-compose exec datanode bash
- - `ozone freon -mode offline -validateWrites -numOfVolumes 1 -numOfBuckets 10 -numOfKeys 100`
+ozone freon -mode offline -validateWrites -numOfVolumes 1 -numOfBuckets 10 -numOfKeys 100
+```
-You can checkout the KSM UI to see the requests information.
+You can checkout the OM UI to see the requests information.
+```
+http://localhost:9874/
+```
- - `http://localhost:9874/`
+If you need more datanodes you can scale up:
+```
+docker-compose up --scale datanode=3 -d
+```
+
+## Running Ozone using a real cluster
-If you need more datanode you can scale up:
+### Configuration
- - `docker-compose scale datanode=3`
+First initialize Hadoop cluster configuration files like hadoop-env.sh,
+core-site.xml, hdfs-site.xml and any other configuration files that are
+needed for your cluster.
-Running Ozone using a real cluster
-----------------------------------
+#### Update hdfs-site.xml
-Please proceed to setup a hadoop cluster by creating the hdfs-site.xml and
-other configuration files that are needed for your cluster.
+The container manager part of Ozone runs inside DataNodes as a pluggable module.
+To activate ozone you should define the service plugin implementation class.
+**Important**: It should be added to the **hdfs-site.xml** as the plugin should
+be activated as part of the normal HDFS Datanode bootstrap.
+```
+<property>
+ <name>dfs.datanode.plugins</name>
+ <value>org.apache.hadoop.ozone.HddsDatanodeService</value>
+</property>
+```
-### Ozone Configuration
+#### Create ozone-site.xml
-Ozone relies on its own configuration file called `ozone-site.xml`. It is
-just for convenience and ease of management -- you can add these settings
-to `hdfs-site.xml`, if you don't want to keep ozone settings separate.
-This document refers to `ozone-site.xml` so that ozone settings are in one
-place and not mingled with HDFS settings.
+Ozone relies on its own configuration file called `ozone-site.xml`.
+The following are the most important settings.
- * _*ozone.enabled*_ This is the most important setting for ozone.
+ 1. _*ozone.enabled*_ This is the most important setting for ozone.
Currently, Ozone is an opt-in subsystem of HDFS. By default, Ozone is
disabled. Setting this flag to `true` enables ozone in the HDFS cluster.
Here is an example,
-
-```
+ ```
<property>
<name>ozone.enabled</name>
<value>True</value>
</property>
-```
- * _*ozone.metadata.dirs*_ Ozone is designed with modern hardware
- in mind. It tries to use SSDs effectively. So users can specify where the
+ ```
+ 1. **ozone.metadata.dirs** Administrators can specify where the
metadata must reside. Usually you pick your fastest disk (SSD if
- you have them on your nodes). KSM, SCM and datanode will write the metadata
+ you have them on your nodes). OM, SCM and datanode will write the metadata
to these disks. This is a required setting, if this is missing Ozone will
fail to come up. Here is an example,
-
-```
+ ```
<property>
<name>ozone.metadata.dirs</name>
<value>/data/disk1/meta</value>
</property>
-```
+ ```
-* _*ozone.scm.names*_ Ozone is build on top of container framework. Storage
+1. **ozone.scm.names** Ozone is build on top of container framework. Storage
container manager(SCM) is a distributed block service which is used by ozone
and other storage services.
This property allows datanodes to discover where SCM is, so that
@@ -136,129 +162,105 @@ place and not mingled with HDFS settings.
and datanodes assume there are multiple instances of SCM which form a highly
available ring. The HA feature of SCM is a work in progress. So we
configure ozone.scm.names to be a single machine. Here is an example,
-
-```
+ ```
<property>
<name>ozone.scm.names</name>
<value>scm.hadoop.apache.org</value>
</property>
-```
+ ```
-* _*ozone.scm.datanode.id*_ Each datanode that speaks to SCM generates an ID
-just like HDFS. This is an optional setting. Please note:
+1. **ozone.scm.datanode.id** Each datanode that speaks to SCM generates an ID
+just like HDFS. This is a mandatory setting. Please note:
This path will be created by datanodes if it doesn't exist already. Here is an
example,
-
-```
+ ```
<property>
<name>ozone.scm.datanode.id</name>
<value>/data/disk1/scm/meta/node/datanode.id</value>
</property>
-```
+ ```
-* _*ozone.scm.block.client.address*_ Storage Container Manager(SCM) offers a
+1. **ozone.scm.block.client.address** Storage Container Manager(SCM) offers a
set of services that can be used to build a distributed storage system. One
- of the services offered is the block services. KSM and HDFS would use this
- service. This property describes where KSM can discover SCM's block service
+ of the services offered is the block services. OM and HDFS would use this
+ service. This property describes where OM can discover SCM's block service
endpoint. There is corresponding ports etc, but assuming that we are using
default ports, the server address is the only required field. Here is an
example,
-
-```
+ ```
<property>
<name>ozone.scm.block.client.address</name>
<value>scm.hadoop.apache.org</value>
</property>
-```
+ ```
-* _*ozone.ksm.address*_ KSM server address. This is used by Ozonehandler and
+1. **ozone.ksm.address** OM server address. This is used by OzoneClient and
Ozone File System.
-
-```
+ ```
<property>
<name>ozone.ksm.address</name>
<value>ksm.hadoop.apache.org</value>
</property>
-```
-
-* _*dfs.datanode.plugin*_ Datanode service plugins: the container manager part
- of ozone is running inside the datanode as a service plugin. To activate ozone
- you should define the service plugin implementation class. **Important**
- It should be added to the **hdfs-site.xml** as the plugin should be activated
- as part of the normal HDFS Datanode bootstrap.
-
-```
- <property>
- <name>dfs.datanode.plugins</name>
- <value>org.apache.hadoop.ozone.HddsDatanodeService</value>
- </property>
-```
+ ```
-Here is a quick summary of settings needed by Ozone.
+#### Ozone Settings Summary
| Setting | Value | Comment |
|--------------------------------|------------------------------|------------------------------------------------------------------|
| ozone.enabled | True | This enables SCM and containers in HDFS cluster. |
| ozone.metadata.dirs | file path | The metadata will be stored here. |
| ozone.scm.names | SCM server name | Hostname:port or or IP:port address of SCM. |
-| ozone.scm.block.client.address | SCM server name and port | Used by services like KSM |
+| ozone.scm.block.client.address | SCM server name and port | Used by services like OM |
| ozone.scm.client.address | SCM server name and port | Used by client side |
| ozone.scm.datanode.address | SCM server name and port | Used by datanode to talk to SCM |
-| ozone.ksm.address | KSM server name | Used by Ozone handler and Ozone file system. |
+| ozone.ksm.address | OM server name | Used by Ozone handler and Ozone file system. |
- Here is a working example of`ozone-site.xml`.
+
+#### Sample ozone-site.xml
```
- <?xml version="1.0" encoding="UTF-8"?>
- <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
- <configuration>
- <property>
- <name>ozone.enabled</name>
- <value>True</value>
- </property>
-
- <property>
- <name>ozone.metadata.dirs</name>
- <value>/data/disk1/ozone/meta</value>
- </property>
-
- <property>
- <name>ozone.scm.names</name>
- <value>127.0.0.1</value>
- </property>
-
- <property>
- <name>ozone.scm.client.address</name>
- <value>127.0.0.1:9860</value>
- </property>
-
- <property>
- <name>ozone.scm.block.client.address</name>
- <value>127.0.0.1:9863</value>
- </property>
-
- <property>
- <name>ozone.scm.datanode.address</name>
- <value>127.0.0.1:9861</value>
- </property>
-
- <property>
- <name>ozone.ksm.address</name>
- <value>127.0.0.1:9874</value>
- </property>
- </configuration>
-```
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>ozone.enabled</name>
+ <value>True</value>
+ </property>
-And don't forget to enable the datanode component with adding the
-following configuration to the hdfs-site.xml:
+ <property>
+ <name>ozone.metadata.dirs</name>
+ <value>/data/disk1/ozone/meta</value>
+ </property>
-```
<property>
- <name>dfs.datanode.plugins</name>
- <value>org.apache.hadoop.ozone.HddsDatanodeService</value>
+ <name>ozone.scm.names</name>
+ <value>127.0.0.1</value>
</property>
+
+ <property>
+ <name>ozone.scm.client.address</name>
+ <value>127.0.0.1:9860</value>
+ </property>
+
+ <property>
+ <name>ozone.scm.block.client.address</name>
+ <value>127.0.0.1:9863</value>
+ </property>
+
+ <property>
+ <name>ozone.scm.datanode.address</name>
+ <value>127.0.0.1:9861</value>
+ </property>
+
+ <property>
+ <name>ozone.ksm.address</name>
+ <value>127.0.0.1:9874</value>
+ </property>
+</configuration>
```
+
+
### Starting Ozone
Ozone is designed to run concurrently with HDFS. The simplest way to [start
@@ -270,35 +272,40 @@ is running, please verify it is fully functional by running some commands like
- *./hdfs dfs -ls /*
Once you are sure that HDFS is running, start Ozone. To start ozone, you
- need to start SCM and KSM. Currently we assume that both KSM and SCM
- is running on the same node, this will change in future.
+ need to start SCM and OM.
- The first time you bring up Ozone, SCM must be initialized.
-
- - `./ozone scm -init`
-
- Start SCM.
-
- - `./ozone --daemon start scm`
-
- Once SCM gets started, KSM must be initialized.
+The first time you bring up Ozone, SCM must be initialized.
+```
+ozone scm -init
+```
- - `./ozone ksm -createObjectStore`
+Start SCM.
+```
+ozone --daemon start scm
+```
- Start KSM.
+Once SCM gets started, OM must be initialized.
+```
+ozone ksm -createObjectStore
+```
- - `./ozone --daemon start ksm`
+Start OM.
+```
+ozone --daemon start ksm
+```
-if you would like to start HDFS and Ozone together, you can do that by running
+If you would like to start HDFS and Ozone together, you can do that by running
a single command.
- - `$HADOOP/sbin/start-ozone.sh`
+```
+$HADOOP/sbin/start-ozone.sh
+```
- This command will start HDFS and then start the ozone components.
+This command will start HDFS and then start the ozone components.
- Once you have ozone running you can use these ozone [shell](./OzoneCommandShell.html)
- commands to create a volume, bucket and keys.
+Once you have ozone running you can use these ozone [shell](./OzoneCommandShell.html)
+commands to start creating a volume, bucket and keys.
-### Diagnosing issues
+## Diagnosing issues
Ozone tries not to pollute the existing HDFS streams of configuration and
logging. So ozone logs are by default configured to be written to a file
@@ -337,16 +344,18 @@ Here is the log4j properties that are added by ozone.
If you would like to have a single datanode log instead of ozone stuff
getting written to ozone.log, please remove this line or set this to true.
+```
+log4j.additivity.org.apache.hadoop.ozone=false
+```
- ` log4j.additivity.org.apache.hadoop.ozone=false`
-
-On the SCM/KSM side, you will be able to see
-
- - `hadoop-hdfs-ksm-hostname.log`
- - `hadoop-hdfs-scm-hostname.log`
+On the SCM/OM side, you will be able to see
+1. `hadoop-hdfs-ksm-hostname.log`
+1. `hadoop-hdfs-scm-hostname.log`
-Please file any issues you see under the related issues:
+## Reporting Bugs
+Please file any issues you see under [Apache HDDS Project Jira](https://issues.apache.org/jira/projects/HDDS/issues/).
+## References
- [Object store in HDFS: HDFS-7240](https://issues.apache.org/jira/browse/HDFS-7240)
- [Ozone File System: HDFS-13074](https://issues.apache.org/jira/browse/HDFS-13074)
- [Building HDFS on top of new storage layer (HDDS): HDFS-10419](https://issues.apache.org/jira/browse/HDFS-10419)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/16] hadoop git commit: Merge branch 'upstream-trunk' into
upstream-HDFS-12943
Posted by su...@apache.org.
Merge branch 'upstream-trunk' into upstream-HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9756c2cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9756c2cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9756c2cb
Branch: refs/heads/HDFS-12943
Commit: 9756c2cbe45b8be4cd16b962b01822d7b006f531
Parents: 990a221 18201b8
Author: Chao Sun <su...@apache.org>
Authored: Mon Jun 11 10:39:34 2018 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Mon Jun 11 10:39:34 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileUtil.java | 6 +
.../rawcoder/AbstractNativeRawDecoder.java | 51 +-
.../rawcoder/AbstractNativeRawEncoder.java | 49 +-
.../rawcoder/NativeRSRawDecoder.java | 19 +-
.../rawcoder/NativeRSRawEncoder.java | 19 +-
.../rawcoder/NativeXORRawDecoder.java | 19 +-
.../rawcoder/NativeXORRawEncoder.java | 19 +-
.../java/org/apache/hadoop/fs/TestFileUtil.java | 52 ++
.../rawcoder/RawErasureCoderBenchmark.java | 6 +
.../util/TestCloseableReferenceCount.java | 91 ++
.../hadoop/util/TestIntrusiveCollection.java | 193 ++++
.../hadoop/util/TestLimitInputStream.java | 74 ++
.../java/org/apache/hadoop/util/TestShell.java | 8 +
.../org/apache/hadoop/util/TestStringUtils.java | 27 +
.../hadoop/util/TestUTF8ByteArrayUtils.java | 57 ++
.../hadoop/hdds/scm/XceiverClientHandler.java | 2 +-
.../org/apache/hadoop/hdds/client/BlockID.java | 2 +-
.../states/endpoint/RegisterEndpointTask.java | 2 +-
.../container/CloseContainerEventHandler.java | 83 ++
.../hdds/scm/container/ContainerMapping.java | 5 +
.../scm/container/ContainerStateManager.java | 9 +
.../hadoop/hdds/scm/container/Mapping.java | 6 +
.../TestCloseContainerEventHandler.java | 177 ++++
.../ha/ConfiguredFailoverProxyProvider.java | 10 +-
...apache.hadoop.security.token.TokenIdentifier | 17 +
...rg.apache.hadoop.security.token.TokenRenewer | 16 +
.../server/blockmanagement/DatanodeManager.java | 2 +-
.../server/namenode/FSDirErasureCodingOp.java | 23 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 10 +-
.../hdfs/server/namenode/FSNamesystem.java | 21 +-
...apache.hadoop.security.token.TokenIdentifier | 17 -
...rg.apache.hadoop.security.token.TokenRenewer | 16 -
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 5 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 16 +
.../hdfs/TestErasureCodingExerciseAPIs.java | 2 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 2 +-
.../hadoop-hdfs/src/test/resources/editsStored | Bin 7909 -> 7909 bytes
.../src/test/resources/editsStored.xml | 2 +-
.../ozone/client/rest/response/KeyInfo.java | 4 +-
.../hadoop/ozone/web/response/KeyInfo.java | 4 +-
hadoop-ozone/docs/content/CommandShell.md | 141 +--
hadoop-ozone/docs/content/GettingStarted.md | 353 +++----
.../TestStorageContainerManagerHelper.java | 2 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 2 +-
.../ozone/ksm/TestContainerReportWithKeys.java | 2 +-
.../hadoop/ozone/ksm/TestKeySpaceManager.java | 2 +-
.../ozone/ksm/TestKsmBlockVersioning.java | 2 +-
.../ksm/TestMultipleContainerReadWrite.java | 2 +-
.../hadoop/ozone/ozShell/TestOzoneShell.java | 2 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 2 +-
.../ozone/scm/TestXceiverClientManager.java | 2 +-
.../ozone/web/TestOzoneRestWithMiniCluster.java | 2 +-
.../hadoop/ozone/web/client/TestKeys.java | 8 +-
.../hadoop/ozone/web/client/TestKeysRatis.java | 2 +-
.../hadoop/ozone/web/client/TestVolume.java | 2 +-
.../hadoop/ozone/ksm/TestChunkStreams.java | 2 +-
.../org/apache/hadoop/ozone/freon/Freon.java | 6 +-
.../genesis/BenchMarkDatanodeDispatcher.java | 2 +-
.../genesis/BenchMarkMetadataStoreReads.java | 2 +-
.../genesis/BenchMarkMetadataStoreWrites.java | 2 +-
.../ozone/genesis/BenchMarkRocksDbStore.java | 2 +-
.../hadoop/ozone/genesis/GenesisUtil.java | 2 +-
hadoop-project/pom.xml | 2 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 21 +
.../containermanager/container/Container.java | 8 +
.../container/ContainerImpl.java | 5 +
.../linux/resources/DefaultOOMHandler.java | 249 +++--
.../impl/container-executor.c | 54 +-
.../impl/container-executor.h | 4 +
.../main/native/container-executor/impl/main.c | 19 +-
.../linux/resources/TestDefaultOOMHandler.java | 922 ++++++++++++++++---
.../nodemanager/webapp/MockContainer.java | 5 +
.../src/site/markdown/Federation.md | 14 +-
.../src/site/markdown/NodeManagerCgroups.md | 2 +-
74 files changed, 2373 insertions(+), 619 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9756c2cb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/16] hadoop git commit: HDFS-13667:Typo: Marking all datandoes as
stale. Contributed by Nanda Kumar
Posted by su...@apache.org.
HDFS-13667:Typo: Marking all datandoes as stale. Contributed by Nanda Kumar
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ccfb816d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ccfb816d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ccfb816d
Branch: refs/heads/HDFS-12943
Commit: ccfb816d39878abf4172933327d788c59b9eb082
Parents: ef0118b
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Sat Jun 9 16:39:09 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Sat Jun 9 16:43:03 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccfb816d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index e6cd513..9ebc693 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -1834,7 +1834,7 @@ public class DatanodeManager {
}
public void markAllDatanodesStale() {
- LOG.info("Marking all datandoes as stale");
+ LOG.info("Marking all datanodes as stale");
synchronized (this) {
for (DatanodeDescriptor dn : datanodeMap.values()) {
for(DatanodeStorageInfo storage : dn.getStorageInfos()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/16] hadoop git commit: HDFS-12670. can't renew HDFS tokens with
only the hdfs client jar. Contributed by Arpit Agarwal.
Posted by su...@apache.org.
HDFS-12670. can't renew HDFS tokens with only the hdfs client jar. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/000a6783
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/000a6783
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/000a6783
Branch: refs/heads/HDFS-12943
Commit: 000a67839666bf7cb39d3955757bb05fa95f1b18
Parents: fba1c42
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Jun 8 17:57:12 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Jun 8 17:57:12 2018 -0700
----------------------------------------------------------------------
...rg.apache.hadoop.security.token.TokenIdentifier | 17 +++++++++++++++++
.../org.apache.hadoop.security.token.TokenRenewer | 16 ++++++++++++++++
...rg.apache.hadoop.security.token.TokenIdentifier | 17 -----------------
.../org.apache.hadoop.security.token.TokenRenewer | 16 ----------------
4 files changed, 33 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/000a6783/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
new file mode 100644
index 0000000..b6b6171
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -0,0 +1,17 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier
+org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier
+org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier$WebHdfsDelegationTokenIdentifier
+org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier$SWebHdfsDelegationTokenIdentifier
http://git-wip-us.apache.org/repos/asf/hadoop/blob/000a6783/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
new file mode 100644
index 0000000..7efd684
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
@@ -0,0 +1,16 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.hadoop.hdfs.DFSClient$Renewer
+org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier$Renewer
+org.apache.hadoop.hdfs.web.TokenAspect$TokenManager
http://git-wip-us.apache.org/repos/asf/hadoop/blob/000a6783/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
deleted file mode 100644
index b6b6171..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier
-org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier
-org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier$WebHdfsDelegationTokenIdentifier
-org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier$SWebHdfsDelegationTokenIdentifier
http://git-wip-us.apache.org/repos/asf/hadoop/blob/000a6783/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
deleted file mode 100644
index 7efd684..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-org.apache.hadoop.hdfs.DFSClient$Renewer
-org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier$Renewer
-org.apache.hadoop.hdfs.web.TokenAspect$TokenManager
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/16] hadoop git commit: HADOOP-15499. Performance severe drops
when running RawErasureCoderBenchmark with NativeRSRawErasureCoder.
Contributed by Sammi Chen.
Posted by su...@apache.org.
HADOOP-15499. Performance severe drops when running RawErasureCoderBenchmark with NativeRSRawErasureCoder. Contributed by Sammi Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18201b88
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18201b88
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18201b88
Branch: refs/heads/HDFS-12943
Commit: 18201b882a38ad875358c5d23c09b0ef903c2f91
Parents: ccfb816
Author: Sammi Chen <sa...@intel.com>
Authored: Mon Jun 11 13:53:37 2018 +0800
Committer: Sammi Chen <sa...@intel.com>
Committed: Mon Jun 11 13:53:37 2018 +0800
----------------------------------------------------------------------
.../rawcoder/AbstractNativeRawDecoder.java | 51 ++++++++++++--------
.../rawcoder/AbstractNativeRawEncoder.java | 49 +++++++++++--------
.../rawcoder/NativeRSRawDecoder.java | 19 ++++++--
.../rawcoder/NativeRSRawEncoder.java | 19 ++++++--
.../rawcoder/NativeXORRawDecoder.java | 19 ++++++--
.../rawcoder/NativeXORRawEncoder.java | 19 ++++++--
.../rawcoder/RawErasureCoderBenchmark.java | 6 +++
7 files changed, 127 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18201b88/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java
index e845747..cb71a80 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java
@@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* Abstract native raw decoder for all native coders to extend with.
@@ -34,36 +35,46 @@ abstract class AbstractNativeRawDecoder extends RawErasureDecoder {
public static Logger LOG =
LoggerFactory.getLogger(AbstractNativeRawDecoder.class);
+ // Protect ISA-L coder data structure in native layer from being accessed and
+ // updated concurrently by the init, release and decode functions.
+ protected final ReentrantReadWriteLock decoderLock =
+ new ReentrantReadWriteLock();
+
public AbstractNativeRawDecoder(ErasureCoderOptions coderOptions) {
super(coderOptions);
}
@Override
- protected synchronized void doDecode(ByteBufferDecodingState decodingState)
+ protected void doDecode(ByteBufferDecodingState decodingState)
throws IOException {
- if (nativeCoder == 0) {
- throw new IOException(String.format("%s closed",
- getClass().getSimpleName()));
- }
- int[] inputOffsets = new int[decodingState.inputs.length];
- int[] outputOffsets = new int[decodingState.outputs.length];
+ decoderLock.readLock().lock();
+ try {
+ if (nativeCoder == 0) {
+ throw new IOException(String.format("%s closed",
+ getClass().getSimpleName()));
+ }
+ int[] inputOffsets = new int[decodingState.inputs.length];
+ int[] outputOffsets = new int[decodingState.outputs.length];
- ByteBuffer buffer;
- for (int i = 0; i < decodingState.inputs.length; ++i) {
- buffer = decodingState.inputs[i];
- if (buffer != null) {
- inputOffsets[i] = buffer.position();
+ ByteBuffer buffer;
+ for (int i = 0; i < decodingState.inputs.length; ++i) {
+ buffer = decodingState.inputs[i];
+ if (buffer != null) {
+ inputOffsets[i] = buffer.position();
+ }
}
- }
- for (int i = 0; i < decodingState.outputs.length; ++i) {
- buffer = decodingState.outputs[i];
- outputOffsets[i] = buffer.position();
- }
+ for (int i = 0; i < decodingState.outputs.length; ++i) {
+ buffer = decodingState.outputs[i];
+ outputOffsets[i] = buffer.position();
+ }
- performDecodeImpl(decodingState.inputs, inputOffsets,
- decodingState.decodeLength, decodingState.erasedIndexes,
- decodingState.outputs, outputOffsets);
+ performDecodeImpl(decodingState.inputs, inputOffsets,
+ decodingState.decodeLength, decodingState.erasedIndexes,
+ decodingState.outputs, outputOffsets);
+ } finally {
+ decoderLock.readLock().unlock();
+ }
}
protected abstract void performDecodeImpl(ByteBuffer[] inputs,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18201b88/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java
index cab5383..44d89c2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java
@@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* Abstract native raw encoder for all native coders to extend with.
@@ -34,34 +35,44 @@ abstract class AbstractNativeRawEncoder extends RawErasureEncoder {
public static Logger LOG =
LoggerFactory.getLogger(AbstractNativeRawEncoder.class);
+ // Protect ISA-L coder data structure in native layer from being accessed and
+ // updated concurrently by the init, release and encode functions.
+ protected final ReentrantReadWriteLock encoderLock =
+ new ReentrantReadWriteLock();
+
public AbstractNativeRawEncoder(ErasureCoderOptions coderOptions) {
super(coderOptions);
}
@Override
- protected synchronized void doEncode(ByteBufferEncodingState encodingState)
+ protected void doEncode(ByteBufferEncodingState encodingState)
throws IOException {
- if (nativeCoder == 0) {
- throw new IOException(String.format("%s closed",
- getClass().getSimpleName()));
- }
- int[] inputOffsets = new int[encodingState.inputs.length];
- int[] outputOffsets = new int[encodingState.outputs.length];
- int dataLen = encodingState.inputs[0].remaining();
+ encoderLock.readLock().lock();
+ try {
+ if (nativeCoder == 0) {
+ throw new IOException(String.format("%s closed",
+ getClass().getSimpleName()));
+ }
+ int[] inputOffsets = new int[encodingState.inputs.length];
+ int[] outputOffsets = new int[encodingState.outputs.length];
+ int dataLen = encodingState.inputs[0].remaining();
- ByteBuffer buffer;
- for (int i = 0; i < encodingState.inputs.length; ++i) {
- buffer = encodingState.inputs[i];
- inputOffsets[i] = buffer.position();
- }
+ ByteBuffer buffer;
+ for (int i = 0; i < encodingState.inputs.length; ++i) {
+ buffer = encodingState.inputs[i];
+ inputOffsets[i] = buffer.position();
+ }
- for (int i = 0; i < encodingState.outputs.length; ++i) {
- buffer = encodingState.outputs[i];
- outputOffsets[i] = buffer.position();
- }
+ for (int i = 0; i < encodingState.outputs.length; ++i) {
+ buffer = encodingState.outputs[i];
+ outputOffsets[i] = buffer.position();
+ }
- performEncodeImpl(encodingState.inputs, inputOffsets, dataLen,
- encodingState.outputs, outputOffsets);
+ performEncodeImpl(encodingState.inputs, inputOffsets, dataLen,
+ encodingState.outputs, outputOffsets);
+ } finally {
+ encoderLock.readLock().unlock();
+ }
}
protected abstract void performEncodeImpl(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18201b88/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java
index 8572222..dc2c33a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java
@@ -36,19 +36,30 @@ public class NativeRSRawDecoder extends AbstractNativeRawDecoder {
public NativeRSRawDecoder(ErasureCoderOptions coderOptions) {
super(coderOptions);
- initImpl(coderOptions.getNumDataUnits(), coderOptions.getNumParityUnits());
+ decoderLock.writeLock().lock();
+ try {
+ initImpl(coderOptions.getNumDataUnits(),
+ coderOptions.getNumParityUnits());
+ } finally {
+ decoderLock.writeLock().unlock();
+ }
}
@Override
- protected synchronized void performDecodeImpl(
+ protected void performDecodeImpl(
ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased,
ByteBuffer[] outputs, int[] outputOffsets) throws IOException {
decodeImpl(inputs, inputOffsets, dataLen, erased, outputs, outputOffsets);
}
@Override
- public synchronized void release() {
- destroyImpl();
+ public void release() {
+ decoderLock.writeLock().lock();
+ try {
+ destroyImpl();
+ } finally {
+ decoderLock.writeLock().unlock();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18201b88/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java
index 754ec88..ad06927 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java
@@ -36,19 +36,30 @@ public class NativeRSRawEncoder extends AbstractNativeRawEncoder {
public NativeRSRawEncoder(ErasureCoderOptions coderOptions) {
super(coderOptions);
- initImpl(coderOptions.getNumDataUnits(), coderOptions.getNumParityUnits());
+ encoderLock.writeLock().lock();
+ try {
+ initImpl(coderOptions.getNumDataUnits(),
+ coderOptions.getNumParityUnits());
+ } finally {
+ encoderLock.writeLock().unlock();
+ }
}
@Override
- protected synchronized void performEncodeImpl(
+ protected void performEncodeImpl(
ByteBuffer[] inputs, int[] inputOffsets, int dataLen,
ByteBuffer[] outputs, int[] outputOffsets) throws IOException {
encodeImpl(inputs, inputOffsets, dataLen, outputs, outputOffsets);
}
@Override
- public synchronized void release() {
- destroyImpl();
+ public void release() {
+ encoderLock.writeLock().lock();
+ try {
+ destroyImpl();
+ } finally {
+ encoderLock.writeLock().unlock();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18201b88/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java
index 1763042..dd708eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java
@@ -36,19 +36,30 @@ public class NativeXORRawDecoder extends AbstractNativeRawDecoder {
public NativeXORRawDecoder(ErasureCoderOptions coderOptions) {
super(coderOptions);
- initImpl(coderOptions.getNumDataUnits(), coderOptions.getNumParityUnits());
+ decoderLock.writeLock().lock();
+ try {
+ initImpl(coderOptions.getNumDataUnits(),
+ coderOptions.getNumParityUnits());
+ } finally {
+ decoderLock.writeLock().unlock();
+ }
}
@Override
- protected synchronized void performDecodeImpl(
+ protected void performDecodeImpl(
ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased,
ByteBuffer[] outputs, int[] outputOffsets) throws IOException {
decodeImpl(inputs, inputOffsets, dataLen, erased, outputs, outputOffsets);
}
@Override
- public synchronized void release() {
- destroyImpl();
+ public void release() {
+ decoderLock.writeLock().lock();
+ try {
+ destroyImpl();
+ } finally {
+ decoderLock.writeLock().unlock();
+ }
}
private native void initImpl(int numDataUnits, int numParityUnits);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18201b88/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java
index 7f4265b..66b0a1b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java
@@ -36,19 +36,30 @@ public class NativeXORRawEncoder extends AbstractNativeRawEncoder {
public NativeXORRawEncoder(ErasureCoderOptions coderOptions) {
super(coderOptions);
- initImpl(coderOptions.getNumDataUnits(), coderOptions.getNumParityUnits());
+ encoderLock.writeLock().lock();
+ try {
+ initImpl(coderOptions.getNumDataUnits(),
+ coderOptions.getNumParityUnits());
+ } finally {
+ encoderLock.writeLock().unlock();
+ }
}
@Override
- protected synchronized void performEncodeImpl(
+ protected void performEncodeImpl(
ByteBuffer[] inputs, int[] inputOffsets, int dataLen,
ByteBuffer[] outputs, int[] outputOffsets) throws IOException {
encodeImpl(inputs, inputOffsets, dataLen, outputs, outputOffsets);
}
@Override
- public synchronized void release() {
- destroyImpl();
+ public void release() {
+ encoderLock.writeLock().lock();
+ try {
+ destroyImpl();
+ } finally {
+ encoderLock.writeLock().unlock();
+ }
}
private native void initImpl(int numDataUnits, int numParityUnits);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18201b88/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
index c005e77..df8c54b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
@@ -230,6 +230,12 @@ public final class RawErasureCoderBenchmark {
throw e;
} finally {
executor.shutdown();
+ if (encoder != null) {
+ encoder.release();
+ }
+ if (decoder != null) {
+ decoder.release();
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/16] hadoop git commit: HADOOP-15520. Add tests for various
org.apache.hadoop.util classes. Contributed by Arash Nabili
Posted by su...@apache.org.
HADOOP-15520. Add tests for various org.apache.hadoop.util classes.
Contributed by Arash Nabili
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef0118b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef0118b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef0118b9
Branch: refs/heads/HDFS-12943
Commit: ef0118b91e384b9a6d96c2ae64480d9acf5aa6fb
Parents: 000a678
Author: Steve Loughran <st...@apache.org>
Authored: Sat Jun 9 15:33:30 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Jun 9 15:33:38 2018 +0100
----------------------------------------------------------------------
.../util/TestCloseableReferenceCount.java | 91 +++++++++
.../hadoop/util/TestIntrusiveCollection.java | 193 +++++++++++++++++++
.../hadoop/util/TestLimitInputStream.java | 74 +++++++
.../java/org/apache/hadoop/util/TestShell.java | 8 +
.../org/apache/hadoop/util/TestStringUtils.java | 27 +++
.../hadoop/util/TestUTF8ByteArrayUtils.java | 57 ++++++
6 files changed, 450 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef0118b9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCloseableReferenceCount.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCloseableReferenceCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCloseableReferenceCount.java
new file mode 100644
index 0000000..31e1899
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCloseableReferenceCount.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.nio.channels.ClosedChannelException;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestCloseableReferenceCount extends HadoopTestBase {
+ @Test
+ public void testReference() throws ClosedChannelException {
+ CloseableReferenceCount clr = new CloseableReferenceCount();
+ clr.reference();
+ assertEquals("Incorrect reference count", 1, clr.getReferenceCount());
+ }
+
+ @Test
+ public void testUnreference() throws ClosedChannelException {
+ CloseableReferenceCount clr = new CloseableReferenceCount();
+ clr.reference();
+ clr.reference();
+ assertFalse("New reference count should not equal STATUS_CLOSED_MASK",
+ clr.unreference());
+ assertEquals("Incorrect reference count", 1, clr.getReferenceCount());
+ }
+
+ @Test
+ public void testUnreferenceCheckClosed() throws ClosedChannelException {
+ CloseableReferenceCount clr = new CloseableReferenceCount();
+ clr.reference();
+ clr.reference();
+ clr.unreferenceCheckClosed();
+ assertEquals("Incorrect reference count", 1, clr.getReferenceCount());
+ }
+
+ @Test
+ public void testSetClosed() throws ClosedChannelException {
+ CloseableReferenceCount clr = new CloseableReferenceCount();
+ assertTrue("Reference count should be open", clr.isOpen());
+ clr.setClosed();
+ assertFalse("Reference count should be closed", clr.isOpen());
+ }
+
+ @Test(expected = ClosedChannelException.class)
+ public void testReferenceClosedReference() throws ClosedChannelException {
+ CloseableReferenceCount clr = new CloseableReferenceCount();
+ clr.setClosed();
+ assertFalse("Reference count should be closed", clr.isOpen());
+ clr.reference();
+ }
+
+ @Test(expected = ClosedChannelException.class)
+ public void testUnreferenceClosedReference() throws ClosedChannelException {
+ CloseableReferenceCount clr = new CloseableReferenceCount();
+ clr.reference();
+ clr.setClosed();
+ assertFalse("Reference count should be closed", clr.isOpen());
+ clr.unreferenceCheckClosed();
+ }
+
+ @Test(expected = ClosedChannelException.class)
+ public void testDoubleClose() throws ClosedChannelException {
+ CloseableReferenceCount clr = new CloseableReferenceCount();
+ assertTrue("Reference count should be open", clr.isOpen());
+ clr.setClosed();
+ assertFalse("Reference count should be closed", clr.isOpen());
+ clr.setClosed();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef0118b9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestIntrusiveCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestIntrusiveCollection.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestIntrusiveCollection.java
new file mode 100644
index 0000000..03bbf7b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestIntrusiveCollection.java
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <pre>
+ * Story 1
+ * As a software developer,
+ * I want to use the IntrusiveCollection class;
+ * So that I can save on memory usage during execution.
+ * </pre>
+ */
+package org.apache.hadoop.util;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.HadoopTestBase;
+import org.apache.hadoop.util.IntrusiveCollection.Element;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestIntrusiveCollection extends HadoopTestBase {
+ static class SimpleElement implements IntrusiveCollection.Element {
+ private Map<IntrusiveCollection<? extends Element>, Element>
+ prevMap, nextMap;
+ private Map<IntrusiveCollection<? extends Element>, Boolean> isMemberMap;
+
+ public SimpleElement() {
+ prevMap = new HashMap<>();
+ nextMap = new HashMap<>();
+ isMemberMap = new HashMap<>();
+ }
+
+ @Override
+ public void insertInternal(IntrusiveCollection<? extends Element> list,
+ Element prev, Element next) {
+ isMemberMap.put(list, true);
+ prevMap.put(list, prev);
+ nextMap.put(list, next);
+ }
+
+ @Override
+ public void setPrev(IntrusiveCollection<? extends Element> list,
+ Element prev) {
+ prevMap.put(list, prev);
+ }
+
+ @Override
+ public void setNext(IntrusiveCollection<? extends Element> list,
+ Element next) {
+ nextMap.put(list, next);
+ }
+
+ @Override
+ public void removeInternal(IntrusiveCollection<? extends Element> list) {
+ prevMap.remove(list);
+ nextMap.remove(list);
+ isMemberMap.remove(list);
+ }
+
+ @Override
+ public Element getPrev(IntrusiveCollection<? extends Element> list) {
+ return prevMap.getOrDefault(list, null);
+ }
+
+ @Override
+ public Element getNext(IntrusiveCollection<? extends Element> list) {
+ return nextMap.getOrDefault(list, null);
+ }
+
+ @Override
+ public boolean isInList(IntrusiveCollection<? extends Element> list) {
+ return isMemberMap.getOrDefault(list, false);
+ }
+ }
+
+ /**
+ * <pre>
+ * Scenario S1.1: Adding an element
+ * Given an IntrusiveCollection has been created
+ * and the IntrusiveCollection is empty
+ * When I insert an element
+ * Then the IntrusiveCollection contains the newly added element.
+ * </pre>
+ */
+ @Test
+ public void testShouldAddElement() {
+ IntrusiveCollection<SimpleElement> intrusiveCollection =
+ new IntrusiveCollection<>();
+
+ SimpleElement element = new SimpleElement();
+ intrusiveCollection.add(element);
+
+ assertFalse("Collection should not be empty",
+ intrusiveCollection.isEmpty());
+ assertTrue("Collection should contain added element",
+ intrusiveCollection.contains(element));
+ }
+
+ /**
+ * <pre>
+ * Scenario S1.2: Removing an element
+ * Given an IntrusiveCollection has been created
+ * and the InstrusiveCollection contains a single element
+ * When I remove the element
+ * Then the IntrusiveCollection is empty.
+ * </pre>
+ */
+ @Test
+ public void testShouldRemoveElement() {
+ IntrusiveCollection<SimpleElement> intrusiveCollection =
+ new IntrusiveCollection<>();
+ SimpleElement element = new SimpleElement();
+ intrusiveCollection.add(element);
+
+ intrusiveCollection.remove(element);
+
+ assertTrue("Collection should be empty", intrusiveCollection.isEmpty());
+ assertFalse("Collection should not contain removed element",
+ intrusiveCollection.contains(element));
+ }
+
+ /**
+ * <pre>
+ * Scenario S1.3: Removing all elements
+ * Given an IntrusiveCollection has been created
+ * and the IntrusiveCollection contains multiple elements
+ * When I remove all elements
+ * Then the IntrusiveCollection is empty.
+ * </pre>
+ */
+ @Test
+ public void testShouldRemoveAllElements() {
+ IntrusiveCollection<SimpleElement> intrusiveCollection =
+ new IntrusiveCollection<>();
+ intrusiveCollection.add(new SimpleElement());
+ intrusiveCollection.add(new SimpleElement());
+ intrusiveCollection.add(new SimpleElement());
+
+ intrusiveCollection.clear();
+
+ assertTrue("Collection should be empty", intrusiveCollection.isEmpty());
+ }
+
+ /**
+ * <pre>
+ * Scenario S1.4: Iterating through elements
+ * Given an IntrusiveCollection has been created
+ * and the IntrusiveCollection contains multiple elements
+ * When I iterate through the IntrusiveCollection
+ * Then I get each element in the collection, successively.
+ * </pre>
+ */
+ @Test
+ public void testIterateShouldReturnAllElements() {
+ IntrusiveCollection<SimpleElement> intrusiveCollection =
+ new IntrusiveCollection<>();
+ SimpleElement elem1 = new SimpleElement();
+ SimpleElement elem2 = new SimpleElement();
+ SimpleElement elem3 = new SimpleElement();
+ intrusiveCollection.add(elem1);
+ intrusiveCollection.add(elem2);
+ intrusiveCollection.add(elem3);
+
+ Iterator<SimpleElement> iterator = intrusiveCollection.iterator();
+
+ assertEquals("First element returned is incorrect", elem1, iterator.next());
+ assertEquals("Second element returned is incorrect", elem2,
+ iterator.next());
+ assertEquals("Third element returned is incorrect", elem3, iterator.next());
+ assertFalse("Iterator should not have next element", iterator.hasNext());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef0118b9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLimitInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLimitInputStream.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLimitInputStream.java
new file mode 100644
index 0000000..368fa37
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLimitInputStream.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Random;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class TestLimitInputStream extends HadoopTestBase {
+ static class RandomInputStream extends InputStream {
+ private Random rn = new Random(0);
+
+ @Override
+ public int read() { return rn.nextInt(); }
+ }
+
+ @Test
+ public void testRead() throws IOException {
+ try (LimitInputStream limitInputStream =
+ new LimitInputStream(new RandomInputStream(), 0)) {
+ assertEquals("Reading byte after reaching limit should return -1", -1,
+ limitInputStream.read());
+ }
+ try (LimitInputStream limitInputStream =
+ new LimitInputStream(new RandomInputStream(), 4)) {
+ assertEquals("Incorrect byte returned", new Random(0).nextInt(),
+ limitInputStream.read());
+ }
+ }
+
+ @Test(expected = IOException.class)
+ public void testResetWithoutMark() throws IOException {
+ try (LimitInputStream limitInputStream =
+ new LimitInputStream(new RandomInputStream(), 128)) {
+ limitInputStream.reset();
+ }
+ }
+
+ @Test
+ public void testReadBytes() throws IOException {
+ try (LimitInputStream limitInputStream =
+ new LimitInputStream(new RandomInputStream(), 128)) {
+ Random r = new Random(0);
+ byte[] data = new byte[4];
+ byte[] expected = { (byte) r.nextInt(), (byte) r.nextInt(),
+ (byte) r.nextInt(), (byte) r.nextInt() };
+ limitInputStream.read(data, 0, 4);
+ assertArrayEquals("Incorrect bytes returned", expected, data);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef0118b9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
index d0ebc2b..578d267 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
@@ -27,6 +27,7 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.io.PrintWriter;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
@@ -38,6 +39,8 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.test.GenericTestUtils;
import static org.apache.hadoop.util.Shell.*;
+import static org.junit.Assume.assumeTrue;
+
import org.junit.Assume;
import org.junit.Before;
import org.junit.Rule;
@@ -528,4 +531,9 @@ public class TestShell extends Assert {
public void testIsJavaVersionAtLeast() {
assertTrue(Shell.isJavaVersionAtLeast(8));
}
+
+ @Test
+ public void testIsBashSupported() throws InterruptedIOException {
+ assumeTrue("Bash is not supported", Shell.checkIsBashSupported());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef0118b9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
index 96a6482..3fdc1bb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
import java.util.ArrayList;
@@ -476,6 +477,32 @@ public class TestStringUtils extends UnitTestcaseTimeLimit {
executorService.awaitTermination(50, TimeUnit.SECONDS);
}
+ @Test
+ public void testFormatTimeSortable() {
+ long timeDiff = 523452311;
+ String timeDiffStr = "99hrs, 59mins, 59sec";
+
+ assertEquals("Incorrect time diff string returned", timeDiffStr,
+ StringUtils.formatTimeSortable(timeDiff));
+ }
+
+ @Test
+ public void testIsAlpha() {
+ assertTrue("Reported hello as non-alpha string",
+ StringUtils.isAlpha("hello"));
+ assertFalse("Reported hello1 as alpha string",
+ StringUtils.isAlpha("hello1"));
+ }
+
+ @Test
+ public void testEscapeHTML() {
+ String htmlStr = "<p>Hello. How are you?</p>";
+ String escapedStr = "<p>Hello. How are you?</p>";
+
+ assertEquals("Incorrect escaped HTML string returned",
+ escapedStr, StringUtils.escapeHTML(htmlStr));
+ }
+
// Benchmark for StringUtils split
public static void main(String []args) {
final String TO_SPLIT = "foo,bar,baz,blah,blah";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef0118b9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestUTF8ByteArrayUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestUTF8ByteArrayUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestUTF8ByteArrayUtils.java
new file mode 100644
index 0000000..3aa549a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestUTF8ByteArrayUtils.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestUTF8ByteArrayUtils extends HadoopTestBase {
+ @Test
+ public void testFindByte() {
+ byte[] data = "Hello, world!".getBytes();
+ assertEquals("Character 'a' does not exist in string", -1,
+ UTF8ByteArrayUtils.findByte(data, 0, data.length, (byte) 'a'));
+ assertEquals("Did not find first occurrence of character 'o'", 4,
+ UTF8ByteArrayUtils.findByte(data, 0, data.length, (byte) 'o'));
+ }
+
+ @Test
+ public void testFindBytes() {
+ byte[] data = "Hello, world!".getBytes();
+ assertEquals("Did not find first occurrence of pattern 'ello'", 1,
+ UTF8ByteArrayUtils.findBytes(data, 0, data.length, "ello".getBytes()));
+ assertEquals(
+ "Substring starting at position 2 does not contain pattern 'ello'", -1,
+ UTF8ByteArrayUtils.findBytes(data, 2, data.length, "ello".getBytes()));
+ }
+
+ @Test
+ public void testFindNthByte() {
+ byte[] data = "Hello, world!".getBytes();
+ assertEquals("Did not find 2nd occurrence of character 'l'", 3,
+ UTF8ByteArrayUtils.findNthByte(data, 0, data.length, (byte) 'l', 2));
+ assertEquals("4th occurrence of character 'l' does not exist", -1,
+ UTF8ByteArrayUtils.findNthByte(data, 0, data.length, (byte) 'l', 4));
+ assertEquals("Did not find 3rd occurrence of character 'l'", 10,
+ UTF8ByteArrayUtils.findNthByte(data, (byte) 'l', 3));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/16] hadoop git commit: HDFS-13642. Creating a file with block
size smaller than EC policy's cell size should fail.
Posted by su...@apache.org.
HDFS-13642. Creating a file with block size smaller than EC policy's cell size should fail.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf410831
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf410831
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf410831
Branch: refs/heads/HDFS-12943
Commit: cf4108313da83e28d07676078a33016ec8856ff6
Parents: a127244
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Jun 8 15:13:38 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Jun 8 15:14:11 2018 -0700
----------------------------------------------------------------------
.../server/namenode/FSDirErasureCodingOp.java | 23 +++++++++++++++----
.../hdfs/server/namenode/FSDirWriteFileOp.java | 10 ++------
.../hdfs/server/namenode/FSNamesystem.java | 21 +++++++++++++----
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 5 ++--
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 16 +++++++++++++
.../hdfs/TestErasureCodingExerciseAPIs.java | 2 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 2 +-
.../hadoop-hdfs/src/test/resources/editsStored | Bin 7909 -> 7909 bytes
.../src/test/resources/editsStored.xml | 2 +-
9 files changed, 58 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf410831/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 3a32db4..7160b86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.XAttr;
@@ -344,16 +345,28 @@ final class FSDirErasureCodingOp {
}
/**
- * Check if the file or directory has an erasure coding policy.
+ * Get the erasure coding policy information for specified path and policy
+ * name. If ec policy name is given, it will be parsed and the corresponding
+ * policy will be returned. Otherwise, get the policy from the parents of the
+ * iip.
*
* @param fsn namespace
+ * @param ecPolicyName the ec policy name
* @param iip inodes in the path containing the file
- * @return Whether the file or directory has an erasure coding policy.
+ * @return {@link ErasureCodingPolicy}, or null if no policy is found
* @throws IOException
*/
- static boolean hasErasureCodingPolicy(final FSNamesystem fsn,
- final INodesInPath iip) throws IOException {
- return unprotectedGetErasureCodingPolicy(fsn, iip) != null;
+ static ErasureCodingPolicy getErasureCodingPolicy(FSNamesystem fsn,
+ String ecPolicyName, INodesInPath iip) throws IOException {
+ ErasureCodingPolicy ecPolicy;
+ if (!StringUtils.isEmpty(ecPolicyName)) {
+ ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicyByName(
+ fsn, ecPolicyName);
+ } else {
+ ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
+ fsn, iip);
+ }
+ return ecPolicy;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf410831/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 8f34e1c..03c349c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.hdfs.AddBlockFlag;
@@ -543,13 +542,8 @@ class FSDirWriteFileOp {
boolean isStriped = false;
ErasureCodingPolicy ecPolicy = null;
if (!shouldReplicate) {
- if (!StringUtils.isEmpty(ecPolicyName)) {
- ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicyByName(
- fsd.getFSNamesystem(), ecPolicyName);
- } else {
- ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
- fsd.getFSNamesystem(), existing);
- }
+ ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(
+ fsd.getFSNamesystem(), ecPolicyName, existing);
if (ecPolicy != null && (!ecPolicy.isReplicationPolicy())) {
isStriped = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf410831/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 19ff08d..a8c1926 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2403,11 +2403,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
iip = FSDirWriteFileOp.resolvePathForStartFile(
dir, pc, src, flag, createParent);
- if (shouldReplicate ||
- (org.apache.commons.lang.StringUtils.isEmpty(ecPolicyName) &&
- !FSDirErasureCodingOp.hasErasureCodingPolicy(this, iip))) {
- blockManager.verifyReplication(src, replication, clientMachine);
- }
if (blockSize < minBlockSize) {
throw new IOException("Specified block size is less than configured" +
@@ -2415,6 +2410,22 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
+ "): " + blockSize + " < " + minBlockSize);
}
+ if (shouldReplicate) {
+ blockManager.verifyReplication(src, replication, clientMachine);
+ } else {
+ final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp
+ .getErasureCodingPolicy(this, ecPolicyName, iip);
+ if (ecPolicy != null && (!ecPolicy.isReplicationPolicy())) {
+ if (blockSize < ecPolicy.getCellSize()) {
+ throw new IOException("Specified block size (" + blockSize
+ + ") is less than the cell size (" + ecPolicy.getCellSize()
+ +") of the erasure coding policy (" + ecPolicy + ").");
+ }
+ } else {
+ blockManager.verifyReplication(src, replication, clientMachine);
+ }
+ }
+
FileEncryptionInfo feInfo = null;
if (!iip.isRaw() && provider != null) {
EncryptionKeyInfo ezInfo = FSDirEncryptionZoneOp.getEncryptionKeyInfo(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf410831/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 7ddd24e..63199f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -1559,8 +1559,9 @@ public class DFSTestUtil {
out.write("replicated".getBytes());
}
- try (FSDataOutputStream out = filesystem.createFile(
- new Path(ecDir, "RS-3-2")).ecPolicyName(ecPolicyRS32.getName()).build()) {
+ try (FSDataOutputStream out = filesystem
+ .createFile(new Path(ecDir, "RS-3-2"))
+ .ecPolicyName(ecPolicyRS32.getName()).blockSize(1024 * 1024).build()) {
out.write("RS-3-2".getBytes());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf410831/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index 3714542..4b9e876 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -221,4 +222,19 @@ public class TestDFSStripedOutputStream {
StripedFileTestUtil.checkData(fs, testPath, writeBytes,
new ArrayList<DatanodeInfo>(), null, blockSize * dataBlocks);
}
+
+ @Test
+ public void testFileBlockSizeSmallerThanCellSize() throws Exception {
+ final Path path = new Path("testFileBlockSizeSmallerThanCellSize");
+ final byte[] bytes = StripedFileTestUtil.generateBytes(cellSize * 2);
+ try {
+ DFSTestUtil.writeFile(fs, path, bytes, cellSize / 2);
+ fail("Creating a file with block size smaller than "
+ + "ec policy's cell size should fail");
+ } catch (IOException expected) {
+ LOG.info("Caught expected exception", expected);
+ GenericTestUtils
+ .assertExceptionContains("less than the cell size", expected);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf410831/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java
index c63ba34..de59a1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java
@@ -71,7 +71,7 @@ public class TestErasureCodingExerciseAPIs {
private DistributedFileSystem fs;
private HdfsAdmin dfsAdmin;
private FileSystemTestWrapper fsWrapper;
- private static final int BLOCK_SIZE = 1 << 14; // 16k
+ private static final int BLOCK_SIZE = 1 << 20; // 1MB
private ErasureCodingPolicy ecPolicy;
private static ErasureCodingPolicy getEcPolicy() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf410831/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 0b7d259..7d97cce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -63,7 +63,7 @@ public class TestErasureCodingPolicies {
private Configuration conf;
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
- private static final int BLOCK_SIZE = 16 * 1024;
+ private static final int BLOCK_SIZE = 1024 * 1024;
private ErasureCodingPolicy ecPolicy;
private FSNamesystem namesystem;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf410831/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
index a0ae78e..0382432 100644
Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf410831/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
index 7e1881c..cc72e0d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
@@ -1510,7 +1510,7 @@
<REPLICATION>1</REPLICATION>
<MTIME>1512607204120</MTIME>
<ATIME>1512607204120</ATIME>
- <BLOCKSIZE>512</BLOCKSIZE>
+ <BLOCKSIZE>1048576</BLOCKSIZE>
<CLIENT_NAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/16] hadoop git commit: Disable mounting cgroups by default
(miklos.szegedi@cloudera.com via rkanter)
Posted by su...@apache.org.
Disable mounting cgroups by default (miklos.szegedi@cloudera.com via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/351cf87c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/351cf87c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/351cf87c
Branch: refs/heads/HDFS-12943
Commit: 351cf87c92872d90f62c476f85ae4d02e485769c
Parents: d5eca1a
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Jun 7 17:09:34 2018 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Jun 7 17:09:34 2018 -0700
----------------------------------------------------------------------
.../impl/container-executor.c | 54 ++++++++++++++------
.../impl/container-executor.h | 4 ++
.../main/native/container-executor/impl/main.c | 19 ++++---
.../src/site/markdown/NodeManagerCgroups.md | 2 +-
4 files changed, 55 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/351cf87c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 1b8842a..baf0e8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -73,6 +73,7 @@ static const char* DEFAULT_BANNED_USERS[] = {"yarn", "mapred", "hdfs", "bin", 0}
static const int DEFAULT_DOCKER_SUPPORT_ENABLED = 0;
static const int DEFAULT_TC_SUPPORT_ENABLED = 0;
+static const int DEFAULT_MOUNT_CGROUP_SUPPORT_ENABLED = 0;
static const char* PROC_PATH = "/proc";
@@ -482,6 +483,12 @@ int is_tc_support_enabled() {
DEFAULT_TC_SUPPORT_ENABLED, &executor_cfg);
}
+int is_mount_cgroups_support_enabled() {
+ return is_feature_enabled(MOUNT_CGROUP_SUPPORT_ENABLED_KEY,
+ DEFAULT_MOUNT_CGROUP_SUPPORT_ENABLED,
+ &executor_cfg);
+}
+
/**
* Utility function to concatenate argB to argA using the concat_pattern.
*/
@@ -2346,20 +2353,25 @@ void chown_dir_contents(const char *dir_path, uid_t uid, gid_t gid) {
DIR *dp;
struct dirent *ep;
- char *path_tmp = malloc(strlen(dir_path) + NAME_MAX + 2);
+ size_t len = strlen(dir_path) + NAME_MAX + 2;
+ char *path_tmp = malloc(len);
if (path_tmp == NULL) {
return;
}
- char *buf = stpncpy(path_tmp, dir_path, strlen(dir_path));
- *buf++ = '/';
-
dp = opendir(dir_path);
if (dp != NULL) {
while ((ep = readdir(dp)) != NULL) {
- stpncpy(buf, ep->d_name, strlen(ep->d_name));
- buf[strlen(ep->d_name)] = '\0';
- change_owner(path_tmp, uid, gid);
+ if (strcmp(ep->d_name, ".") != 0 &&
+ strcmp(ep->d_name, "..") != 0 &&
+ strstr(ep->d_name, "..") == NULL) {
+ int result = snprintf(path_tmp, len, "%s/%s", dir_path, ep->d_name);
+ if (result > 0 && result < len) {
+ change_owner(path_tmp, uid, gid);
+ } else {
+ fprintf(LOGFILE, "Ignored %s/%s due to length", dir_path, ep->d_name);
+ }
+ }
}
closedir(dp);
}
@@ -2383,11 +2395,16 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
char *mount_path = malloc(len);
char hier_path[EXECUTOR_PATH_MAX];
int result = 0;
- struct stat sb;
if (controller == NULL || mount_path == NULL) {
fprintf(LOGFILE, "Failed to mount cgroup controller; not enough memory\n");
result = OUT_OF_MEMORY;
+ goto cleanup;
+ }
+ if (hierarchy == NULL || strstr(hierarchy, "..") != NULL) {
+ fprintf(LOGFILE, "Unsupported cgroup hierarhy path detected.\n");
+ result = INVALID_COMMAND_PROVIDED;
+ goto cleanup;
}
if (get_kv_key(pair, controller, len) < 0 ||
get_kv_value(pair, mount_path, len) < 0) {
@@ -2395,13 +2412,10 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
pair);
result = -1;
} else {
- if (stat(mount_path, &sb) != 0) {
- // Create mount point, if it does not exist
- const mode_t mount_perms = S_IRWXU | S_IRGRP | S_IXGRP;
- if (mkdirs(mount_path, mount_perms) == 0) {
- fprintf(LOGFILE, "Failed to create cgroup mount point %s at %s\n",
- controller, mount_path);
- }
+ if (strstr(mount_path, "..") != NULL) {
+ fprintf(LOGFILE, "Unsupported cgroup mount path detected.\n");
+ result = INVALID_COMMAND_PROVIDED;
+ goto cleanup;
}
if (mount("none", mount_path, "cgroup", 0, controller) == 0) {
char *buf = stpncpy(hier_path, mount_path, strlen(mount_path));
@@ -2410,13 +2424,20 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
// create hierarchy as 0750 and chown to Hadoop NM user
const mode_t perms = S_IRWXU | S_IRGRP | S_IXGRP;
+ struct stat sb;
+ if (stat(hier_path, &sb) == 0 &&
+ (sb.st_uid != nm_uid || sb.st_gid != nm_gid)) {
+ fprintf(LOGFILE, "cgroup hierarchy %s already owned by another user %d\n", hier_path, sb.st_uid);
+ result = INVALID_COMMAND_PROVIDED;
+ goto cleanup;
+ }
if (mkdirs(hier_path, perms) == 0) {
change_owner(hier_path, nm_uid, nm_gid);
chown_dir_contents(hier_path, nm_uid, nm_gid);
}
} else {
fprintf(LOGFILE, "Failed to mount cgroup controller %s at %s - %s\n",
- controller, mount_path, strerror(errno));
+ controller, mount_path, strerror(errno));
// if controller is already mounted, don't stop trying to mount others
if (errno != EBUSY) {
result = -1;
@@ -2424,6 +2445,7 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
}
}
+cleanup:
free(controller);
free(mount_path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/351cf87c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index 9136606..32e953d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -64,6 +64,7 @@ enum operations {
#define ALLOWED_SYSTEM_USERS_KEY "allowed.system.users"
#define DOCKER_SUPPORT_ENABLED_KEY "feature.docker.enabled"
#define TC_SUPPORT_ENABLED_KEY "feature.tc.enabled"
+#define MOUNT_CGROUP_SUPPORT_ENABLED_KEY "feature.mount-cgroup.enabled"
#define TMP_DIR "tmp"
extern struct passwd *user_detail;
@@ -238,6 +239,9 @@ int is_feature_enabled(const char* feature_key, int default_value,
/** Check if tc (traffic control) support is enabled in configuration. */
int is_tc_support_enabled();
+/** Check if cgroup mount support is enabled in configuration. */
+int is_mount_cgroups_support_enabled();
+
/**
* Run a batch of tc commands that modify interface configuration
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/351cf87c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index c54fd3e..2099ace 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -262,14 +262,19 @@ static int validate_arguments(int argc, char **argv , int *operation) {
}
if (strcmp("--mount-cgroups", argv[1]) == 0) {
- if (argc < 4) {
- display_usage(stdout);
- return INVALID_ARGUMENT_NUMBER;
+ if (is_mount_cgroups_support_enabled()) {
+ if (argc < 4) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ optind++;
+ cmd_input.cgroups_hierarchy = argv[optind++];
+ *operation = MOUNT_CGROUPS;
+ return 0;
+ } else {
+ display_feature_disabled_message("mount cgroup");
+ return FEATURE_DISABLED;
}
- optind++;
- cmd_input.cgroups_hierarchy = argv[optind++];
- *operation = MOUNT_CGROUPS;
- return 0;
}
if (strcmp("--tc-modify-state", argv[1]) == 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/351cf87c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md
index d362801..4a83dce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md
@@ -50,7 +50,7 @@ YARN uses CGroups through a directory structure mounted into the file system by
| Option | Description |
|:---- |:---- |
| Discover CGroups mounted already | This should be used on newer systems like RHEL7 or Ubuntu16 or if the administrator mounts CGroups before YARN starts. Set `yarn.nodemanager.linux-container-executor.cgroups.mount` to false and leave other settings set to their defaults. YARN will locate the mount points in `/proc/mounts`. Common locations include `/sys/fs/cgroup` and `/cgroup`. The default location can vary depending on the Linux distribution in use.|
-| CGroups mounted by YARN | If the system does not have CGroups mounted or it is mounted to an inaccessible location then point `yarn.nodemanager.linux-container-executor.cgroups.mount-path` to an empty directory. Set `yarn.nodemanager.linux-container-executor.cgroups.mount` to true. A point to note here is that the container-executor binary will try to create and mount each subsystem as a subdirectory under this path. If `cpu` is already mounted somewhere with `cpuacct`, then the directory `cpu,cpuacct` will be created for the hierarchy.|
+| CGroups mounted by YARN | IMPORTANT: This option is deprecated due to security reasons with the `container-executor.cfg` option `feature.mount-cgroup.enabled=0` by default. Please mount cgroups before launching YARN.|
| CGroups mounted already or linked but not in `/proc/mounts` | If cgroups is accessible through lxcfs or simulated by another filesystem, then point `yarn.nodemanager.linux-container-executor.cgroups.mount-path` to your CGroups root directory. Set `yarn.nodemanager.linux-container-executor.cgroups.mount` to false. YARN tries to use this path first, before any CGroup mount point discovery. The path should have a subdirectory for each CGroup hierarchy named by the comma separated CGroup subsystems supported like `<path>/cpu,cpuacct`. Valid subsystem names are `cpu, cpuacct, cpuset, memory, net_cls, blkio, freezer, devices`.|
CGroups and security
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/16] hadoop git commit: HDDS-157. Upgrade common-langs version to
3.7 in HDDS and Ozone. Contributed by Takanobu Asanuma.
Posted by su...@apache.org.
HDDS-157. Upgrade common-langs version to 3.7 in HDDS and Ozone.
Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1272448
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1272448
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1272448
Branch: refs/heads/HDFS-12943
Commit: a1272448bfa2f1a159d948b8635558e053b7be78
Parents: c42dcc7
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Jun 8 10:27:01 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Fri Jun 8 10:27:01 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/scm/XceiverClientHandler.java | 2 +-
.../src/main/java/org/apache/hadoop/hdds/client/BlockID.java | 2 +-
.../common/states/endpoint/RegisterEndpointTask.java | 2 +-
.../apache/hadoop/ozone/client/rest/response/KeyInfo.java | 4 ++--
.../java/org/apache/hadoop/ozone/web/response/KeyInfo.java | 4 ++--
.../hadoop/ozone/TestStorageContainerManagerHelper.java | 2 +-
.../apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java | 2 +-
.../apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java | 2 +-
.../org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java | 2 +-
.../org/apache/hadoop/ozone/ksm/TestKsmBlockVersioning.java | 2 +-
.../hadoop/ozone/ksm/TestMultipleContainerReadWrite.java | 2 +-
.../java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java | 2 +-
.../org/apache/hadoop/ozone/scm/TestAllocateContainer.java | 2 +-
.../apache/hadoop/ozone/scm/TestXceiverClientManager.java | 2 +-
.../hadoop/ozone/web/TestOzoneRestWithMiniCluster.java | 2 +-
.../java/org/apache/hadoop/ozone/web/client/TestKeys.java | 8 ++++----
.../org/apache/hadoop/ozone/web/client/TestKeysRatis.java | 2 +-
.../java/org/apache/hadoop/ozone/web/client/TestVolume.java | 2 +-
.../java/org/apache/hadoop/ozone/ksm/TestChunkStreams.java | 2 +-
.../src/main/java/org/apache/hadoop/ozone/freon/Freon.java | 6 +++---
.../hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java | 2 +-
.../hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java | 2 +-
.../hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java | 2 +-
.../apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java | 2 +-
.../java/org/apache/hadoop/ozone/genesis/GenesisUtil.java | 2 +-
25 files changed, 32 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java
index 6a2286c..7c568f6 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java
@@ -21,7 +21,7 @@ import com.google.common.base.Preconditions;
import org.apache.ratis.shaded.io.netty.channel.Channel;
import org.apache.ratis.shaded.io.netty.channel.ChannelHandlerContext;
import org.apache.ratis.shaded.io.netty.channel.SimpleChannelInboundHandler;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java
index 7bf8f01..62b12e3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java
@@ -16,7 +16,7 @@
*/
package org.apache.hadoop.hdds.client;
-import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
index 12b48ab..b3d2b62 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
@@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.container.common.states.endpoint;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.container.common.statemachine
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
index 2e1df5f..61c2abb 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
@@ -22,8 +22,8 @@ import java.io.IOException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
/**
* KeyInfo class is used used for parsing json response
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
index 34885f6..ba47bee 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
@@ -19,8 +19,8 @@ package org.apache.hadoop.ozone.web.response;
import java.io.IOException;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
index e5873a5..99e69ec 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
@@ -20,7 +20,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.primitives.Longs;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
index e3823b3..cafe5db 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.client.rpc;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.ozone.MiniOzoneCluster;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java
index 18482d1..bafba32 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java
@@ -16,7 +16,7 @@
*/
package org.apache.hadoop.ozone.ksm;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
index 9c0a488..f07a97d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
@@ -17,7 +17,7 @@
package org.apache.hadoop.ozone.ksm;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKsmBlockVersioning.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKsmBlockVersioning.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKsmBlockVersioning.java
index addd87b..15c3fd3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKsmBlockVersioning.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKsmBlockVersioning.java
@@ -16,7 +16,7 @@
*/
package org.apache.hadoop.ozone.ksm;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtil;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestMultipleContainerReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestMultipleContainerReadWrite.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestMultipleContainerReadWrite.java
index 765ec3f..1cb6e82 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestMultipleContainerReadWrite.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestMultipleContainerReadWrite.java
@@ -16,7 +16,7 @@
*/
package org.apache.hadoop.ozone.ksm;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
index 8ffe67d..0edfd41 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
@@ -38,7 +38,7 @@ import java.util.Random;
import java.util.UUID;
import java.util.stream.Collectors;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java
index ef6fd5f..b1e9d26 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.ozone.scm;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientManager.java
index 77e4e1b..478cf69 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientManager.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.scm;
import com.google.common.cache.Cache;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.io.IOUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
index a9b8175..0f49ade 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
@@ -24,7 +24,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.CHUNK_SIZE;
import static org.junit.Assert.*;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index f8df7dc..28a138e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@ -23,8 +23,8 @@ import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.collections.IteratorUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang.math.RandomUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.client.ReplicationFactor;
@@ -172,10 +172,10 @@ public class TestKeys {
* @return Key composed of multiple parts delimited by "/"
*/
static String getMultiPartKey(String delimiter) {
- int numParts = RandomUtils.nextInt(5) + 1;
+ int numParts = RandomUtils.nextInt(0, 5) + 1;
String[] nameParts = new String[numParts];
for (int i = 0; i < numParts; i++) {
- int stringLength = numParts == 1 ? 5 : RandomUtils.nextInt(5);
+ int stringLength = numParts == 1 ? 5 : RandomUtils.nextInt(0, 5);
nameParts[i] = RandomStringUtils.randomAlphanumeric(stringLength);
}
return StringUtils.join(delimiter, nameParts);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
index 645b866..6a92022 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.ozone.web.client;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.RatisTestHelper;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
index a510430..14fc5e3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.ozone.web.client;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/ksm/TestChunkStreams.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/ksm/TestChunkStreams.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/ksm/TestChunkStreams.java
index de4a85a..e6158bd 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/ksm/TestChunkStreams.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/ksm/TestChunkStreams.java
@@ -16,7 +16,7 @@
*/
package org.apache.hadoop.ozone.ksm;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
import org.apache.hadoop.hdds.scm.storage.ChunkInputStream;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
index d933e6f..ab52b86 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
@@ -29,9 +29,9 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang.time.DurationFormatUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.time.DurationFormatUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hdds.client.OzoneQuota;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index a5d268d..13b04c3 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.genesis;
import org.apache.ratis.shaded.com.google.protobuf.ByteString;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.client.BlockID;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java
index fc3dbcb..05e7920 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.ozone.genesis;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.utils.MetadataStore;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Param;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java
index f496a7d..4321287 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.ozone.genesis;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.utils.MetadataStore;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Param;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java
index c4c6f9e..5f4e035 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.genesis;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.utils.MetadataStore;
import org.apache.hadoop.utils.RocksDBStore;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1272448/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
index 7f864ae..f499925 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.ozone.genesis;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/16] hadoop git commit: YARN-8400. Fix typos in YARN Federation
documentation page. Contributed by Giovanni Matteo Fumarola.
Posted by su...@apache.org.
YARN-8400. Fix typos in YARN Federation documentation page. Contributed by Giovanni Matteo Fumarola.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67fc70e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67fc70e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67fc70e0
Branch: refs/heads/HDFS-12943
Commit: 67fc70e09f941e9b43b022d9f42a9486ad759e6e
Parents: 78761e8
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Jun 7 16:10:15 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Jun 7 16:10:15 2018 -0700
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/Federation.md | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67fc70e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
index 087a5b0..953f826 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
@@ -42,7 +42,7 @@ The applications running in this federated environment see a unified large YARN
###YARN Sub-cluster
-A sub-cluster is a YARN cluster with up to few thousands nodes. The exact size of the sub-cluster will be determined considering ease of deployment/maintenance, alignment
+A sub-cluster is a YARN cluster with up to a few thousand nodes. The exact size of the sub-cluster will be determined considering ease of deployment/maintenance, alignment
with network or availability zones and general best practices.
The sub-cluster YARN RM will run with work-preserving high-availability turned-on, i.e., we should be able to tolerate YARN RM, NM failures with minimal disruption.
@@ -80,7 +80,7 @@ to minimize overhead on the scheduling infrastructure (more in section on scalab
###Global Policy Generator
Global Policy Generator overlooks the entire federation and ensures that the system is configured and tuned properly all the time.
-A key design point is that the cluster availability does not depends on an always-on GPG. The GPG operates continuously but out-of-band from all cluster operations,
+A key design point is that the cluster availability does not depend on an always-on GPG. The GPG operates continuously but out-of-band from all cluster operations,
and provide us with a unique vantage point, that allows to enforce global invariants, affect load balancing, trigger draining of sub-clusters that will undergo maintenance, etc.
More precisely the GPG will update user capacity allocation-to-subcluster mappings, and more rarely change the policies that run in Routers, AMRMProxy (and possible RMs).
@@ -111,7 +111,7 @@ on the home sub-cluster. Only in certain cases it should need to ask for resourc
The federation Policy Store is a logically separate store (while it might be backed
by the same physical component), which contains information about how applications and
resource requests are routed to different sub-clusters. The current implementation provides
-several policies, ranging from random/hashing/roundrobin/priority to more sophisticated
+several policies, ranging from random/hashing/round-robin/priority to more sophisticated
ones which account for sub-cluster load, and request locality needs.
@@ -218,7 +218,7 @@ SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**.
|`yarn.federation.policy-manager` | `org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager` | The choice of policy manager determines how Applications and ResourceRequests are routed through the system. |
|`yarn.federation.policy-manager-params` | `<binary>` | The payload that configures the policy. In our example a set of weights for router and amrmproxy policies. This is typically generated by serializing a policymanager that has been configured programmatically, or by populating the state-store with the .json serialized form of it. |
|`yarn.federation.subcluster-resolver.class` | `org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl` | The class used to resolve which subcluster a node belongs to, and which subcluster(s) a rack belongs to. |
-|`yarn.federation.machine-list` | `<path of macihne-list file>` | Path of machine-list file used by `SubClusterResolver`. Each line of the file is a node with sub-cluster and rack information. Below is the example: <br/> <br/> node1, subcluster1, rack1 <br/> node2, subcluster2, rack1 <br/> node3, subcluster3, rack2 <br/> node4, subcluster3, rack2 |
+|`yarn.federation.machine-list` | `<path of machine-list file>` | Path of machine-list file used by `SubClusterResolver`. Each line of the file is a node with sub-cluster and rack information. Below is the example: <br/> <br/> node1, subcluster1, rack1 <br/> node2, subcluster2, rack1 <br/> node3, subcluster3, rack2 <br/> node4, subcluster3, rack2 |
###ON RMs:
@@ -242,7 +242,7 @@ These are extra configurations that should appear in the **conf/yarn-site.xml**
| Property | Example | Description |
|:---- |:---- |
|`yarn.router.bind-host` | `0.0.0.0` | Host IP to bind the router to. The actual address the server will bind to. If this optional address is set, the RPC and webapp servers will bind to this address and the port specified in yarn.router.*.address respectively. This is most useful for making Router listen to all interfaces by setting to 0.0.0.0. |
-| `yarn.router.clientrm.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor` | A comma-seperated list of interceptor classes to be run at the router when interfacing with the client. The last step of this pipeline must be the Federation Client Interceptor. |
+| `yarn.router.clientrm.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor` | A comma-separated list of interceptor classes to be run at the router when interfacing with the client. The last step of this pipeline must be the Federation Client Interceptor. |
Optional:
@@ -273,13 +273,13 @@ Optional:
| Property | Example | Description |
|:---- |:---- |
-| `yarn.nodemanager.amrmproxy.ha.enable` | `true` | Whether or not the AMRMProxy HA is enabled for multiple application attempt suppport. |
+| `yarn.nodemanager.amrmproxy.ha.enable` | `true` | Whether or not the AMRMProxy HA is enabled for multiple application attempt support. |
| `yarn.federation.statestore.max-connections` | `1` | The maximum number of parallel connections from each AMRMProxy to the state-store. This value is typically lower than the router one, since we have many AMRMProxy that could burn-through many DB connections quickly. |
| `yarn.federation.cache-ttl.secs` | `300` | The time to leave for the AMRMProxy cache. Typically larger than at the router, as the number of AMRMProxy is large, and we want to limit the load to the centralized state-store. |
Running a Sample Job
--------------------
-In order to submit jobs to a Federation cluster one must create a seperate set of configs for the client from which jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations:
+In order to submit jobs to a Federation cluster one must create a separate set of configs for the client from which jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations:
| Property | Example | Description |
|:--- |:--- |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/16] hadoop git commit: YARN-6677. Preempt opportunistic
containers when root container cgroup goes over memory limit. Contributed by
Haibo Chen.
Posted by su...@apache.org.
YARN-6677. Preempt opportunistic containers when root container cgroup goes over memory limit. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5eca1a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5eca1a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5eca1a6
Branch: refs/heads/HDFS-12943
Commit: d5eca1a6a0e3939eead6711805b7a61c364d254b
Parents: 67fc70e
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Thu Jun 7 14:58:56 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Thu Jun 7 16:38:23 2018 -0700
----------------------------------------------------------------------
.../containermanager/container/Container.java | 8 +
.../container/ContainerImpl.java | 5 +
.../linux/resources/DefaultOOMHandler.java | 249 +++--
.../linux/resources/TestDefaultOOMHandler.java | 922 ++++++++++++++++---
.../nodemanager/webapp/MockContainer.java | 5 +
5 files changed, 989 insertions(+), 200 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5eca1a6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index 86f2554..5d48d84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -37,8 +37,16 @@ public interface Container extends EventHandler<ContainerEvent> {
ContainerId getContainerId();
+ /**
+ * The timestamp when the container start request is received.
+ */
long getContainerStartTime();
+ /**
+ * The timestamp when the container is allowed to be launched.
+ */
+ long getContainerLaunchTime();
+
Resource getResource();
ContainerTokenIdentifier getContainerTokenIdentifier();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5eca1a6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 5527ac4..95ab374 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -883,6 +883,11 @@ public class ContainerImpl implements Container {
}
@Override
+ public long getContainerLaunchTime() {
+ return this.containerLaunchStartTime;
+ }
+
+ @Override
public Resource getResource() {
return Resources.clone(
this.containerTokenIdentifier.getResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5eca1a6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/DefaultOOMHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/DefaultOOMHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/DefaultOOMHandler.java
index c690225..202e7d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/DefaultOOMHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/DefaultOOMHandler.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p>
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,10 +18,12 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -30,7 +32,7 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Comparator;
+import java.util.Collections;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_FILE_TASKS;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES;
@@ -46,66 +48,60 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
public class DefaultOOMHandler implements Runnable {
protected static final Log LOG = LogFactory
.getLog(DefaultOOMHandler.class);
- private Context context;
- private boolean virtual;
- private CGroupsHandler cgroups;
+ private final Context context;
+ private final String memoryStatFile;
+ private final CGroupsHandler cgroups;
/**
* Create an OOM handler.
* This has to be public to be able to construct through reflection.
* @param context node manager context to work with
- * @param testVirtual Test virtual memory or physical
+ * @param enforceVirtualMemory true if virtual memory needs to be checked,
+ * false if physical memory needs to be checked instead
*/
- public DefaultOOMHandler(Context context, boolean testVirtual) {
+ public DefaultOOMHandler(Context context, boolean enforceVirtualMemory) {
this.context = context;
- this.virtual = testVirtual;
- this.cgroups = ResourceHandlerModule.getCGroupsHandler();
+ this.memoryStatFile = enforceVirtualMemory ?
+ CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES :
+ CGROUP_PARAM_MEMORY_USAGE_BYTES;
+ this.cgroups = getCGroupsHandler();
}
@VisibleForTesting
- void setCGroupsHandler(CGroupsHandler handler) {
- cgroups = handler;
+ protected CGroupsHandler getCGroupsHandler() {
+ return ResourceHandlerModule.getCGroupsHandler();
}
/**
- * Kill the container, if it has exceeded its request.
- *
- * @param container Container to check
- * @param fileName CGroup filename (physical or swap/virtual)
- * @return true, if the container was preempted
+ * Check if a given container exceeds its limits.
*/
- private boolean killContainerIfOOM(Container container, String fileName) {
+ private boolean isContainerOutOfLimit(Container container) {
+ boolean outOfLimit = false;
+
String value = null;
try {
value = cgroups.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- container.getContainerId().toString(),
- fileName);
+ container.getContainerId().toString(), memoryStatFile);
long usage = Long.parseLong(value);
long request = container.getResource().getMemorySize() * 1024 * 1024;
// Check if the container has exceeded its limits.
if (usage > request) {
- // Kill the container
- // We could call the regular cleanup but that sends a
- // SIGTERM first that cannot be handled by frozen processes.
- // Walk through the cgroup
- // tasks file and kill all processes in it
- sigKill(container);
+ outOfLimit = true;
String message = String.format(
- "Container %s was killed by elastic cgroups OOM handler using %d " +
+ "Container %s is out of its limits, using %d " +
"when requested only %d",
container.getContainerId(), usage, request);
LOG.warn(message);
- return true;
}
} catch (ResourceHandlerException ex) {
LOG.warn(String.format("Could not access memory resource for %s",
container.getContainerId()), ex);
} catch (NumberFormatException ex) {
- LOG.warn(String.format("Could not parse %s in %s",
- value, container.getContainerId()));
+ LOG.warn(String.format("Could not parse %s in %s", value,
+ container.getContainerId()));
}
- return false;
+ return outOfLimit;
}
/**
@@ -168,21 +164,16 @@ public class DefaultOOMHandler implements Runnable {
/**
* It is called when the node is under an OOM condition. All processes in
* all sub-cgroups are suspended. We need to act fast, so that we do not
- * affect the overall system utilization.
- * In general we try to find a newly run container that exceeded its limits.
- * The justification is cost, since probably this is the one that has
- * accumulated the least amount of uncommitted data so far.
- * We continue the process until the OOM is resolved.
+ * affect the overall system utilization. In general we try to find a
+ * newly launched container that exceeded its limits. The justification is
+ * cost, since probably this is the one that has accumulated the least
+ * amount of uncommitted data so far. OPPORTUNISTIC containers are always
+ * killed before any GUARANTEED containers are considered. We continue the
+ * process until the OOM is resolved.
*/
@Override
public void run() {
try {
- // Reverse order by start time
- Comparator<Container> comparator = (Container o1, Container o2) -> {
- long order = o1.getContainerStartTime() - o2.getContainerStartTime();
- return order > 0 ? -1 : order < 0 ? 1 : 0;
- };
-
// We kill containers until the kernel reports the OOM situation resolved
// Note: If the kernel has a delay this may kill more than necessary
while (true) {
@@ -194,61 +185,135 @@ public class DefaultOOMHandler implements Runnable {
break;
}
- // The first pass kills a recent container
- // that uses more than its request
- ArrayList<Container> containers = new ArrayList<>();
- containers.addAll(context.getContainers().values());
- // Note: Sorting may take a long time with 10K+ containers
- // but it is acceptable now with low number of containers per node
- containers.sort(comparator);
-
- // Kill the latest container that exceeded its request
- boolean found = false;
- for (Container container : containers) {
- if (!virtual) {
- if (killContainerIfOOM(container,
- CGROUP_PARAM_MEMORY_USAGE_BYTES)) {
- found = true;
- break;
- }
- } else {
- if (killContainerIfOOM(container,
- CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES)) {
- found = true;
- break;
- }
- }
- }
- if (found) {
- continue;
- }
+ boolean containerKilled = killContainer();
- // We have not found any containers that ran out of their limit,
- // so we will kill the latest one. This can happen, if all use
- // close to their request and one of them requests a big block
- // triggering the OOM freeze.
- // Currently there is no other way to identify the outstanding one.
- if (containers.size() > 0) {
- Container container = containers.get(0);
- sigKill(container);
- String message = String.format(
- "Newest container %s killed by elastic cgroups OOM handler using",
- container.getContainerId());
- LOG.warn(message);
- continue;
+ if (!containerKilled) {
+ // This can happen, if SIGKILL did not clean up
+ // non-PGID or containers or containers launched by other users
+ // or if a process was put to the root YARN cgroup.
+ throw new YarnRuntimeException(
+ "Could not find any containers but CGroups " +
+ "reserved for containers ran out of memory. " +
+ "I am giving up");
}
-
- // This can happen, if SIGKILL did not clean up
- // non-PGID or containers or containers launched by other users
- // or if a process was put to the root YARN cgroup.
- throw new YarnRuntimeException(
- "Could not find any containers but CGroups " +
- "reserved for containers ran out of memory. " +
- "I am giving up");
}
} catch (ResourceHandlerException ex) {
- LOG.warn("Could not fecth OOM status. " +
+ LOG.warn("Could not fetch OOM status. " +
"This is expected at shutdown. Exiting.", ex);
}
}
+
+ /**
+ * Choose and kill a container in case of OOM. We try to find the most
+ * recently launched OPPORTUNISTIC container that exceeds its limit
+ * and fall back to the most recently launched OPPORTUNISTIC container
+ * If there is no such container found, we choose to kill a GUARANTEED
+ * container in the same way.
+ * @return true if a container is killed, false otherwise
+ */
+ protected boolean killContainer() {
+ boolean containerKilled = false;
+
+ ArrayList<ContainerCandidate> candidates = new ArrayList<>(0);
+ for (Container container : context.getContainers().values()) {
+ candidates.add(
+ new ContainerCandidate(container, isContainerOutOfLimit(container)));
+ }
+ Collections.sort(candidates);
+
+ if (candidates.size() > 0) {
+ ContainerCandidate candidate = candidates.get(0);
+ sigKill(candidate.container);
+ String message = String.format(
+ "container %s killed by elastic cgroups OOM handler.",
+ candidate.container.getContainerId());
+ LOG.warn(message);
+ containerKilled = true;
+ }
+ return containerKilled;
+ }
+
+ /**
+ * Note: this class has a natural ordering that is inconsistent with equals.
+ */
+ private static class ContainerCandidate
+ implements Comparable<ContainerCandidate> {
+ private final boolean outOfLimit;
+ final Container container;
+
+ ContainerCandidate(Container container, boolean outOfLimit) {
+ this.outOfLimit = outOfLimit;
+ this.container = container;
+ }
+
+ /**
+ * Order two containers by their execution type, followed by
+ * their out-of-limit status and then launch time. Opportunistic
+ * containers are ordered before Guaranteed containers. If two
+ * containers are of the same execution type, the one that is
+ * out of its limits is ordered before the one that isn't. If
+ * two containers have the same execution type and out-of-limit
+ * status, the one that's launched later is ordered before the
+ * other one.
+ */
+ @Override
+ public int compareTo(ContainerCandidate o) {
+ boolean isThisOpportunistic = isOpportunistic(container);
+ boolean isOtherOpportunistic = isOpportunistic(o.container);
+ int ret = Boolean.compare(isOtherOpportunistic, isThisOpportunistic);
+ if (ret == 0) {
+ // the two containers are of the same execution type, order them
+ // by their out-of-limit status.
+ int outOfLimitRet = Boolean.compare(o.outOfLimit, outOfLimit);
+ if (outOfLimitRet == 0) {
+ // the two containers are also of the same out-of-limit status,
+ // order them by their launch time
+ ret = Long.compare(o.container.getContainerLaunchTime(),
+ this.container.getContainerLaunchTime());
+ } else {
+ ret = outOfLimitRet;
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (this.getClass() != obj.getClass()) {
+ return false;
+ }
+ ContainerCandidate other = (ContainerCandidate) obj;
+ if (this.outOfLimit != other.outOfLimit) {
+ return false;
+ }
+ if (this.container == null) {
+ return other.container == null;
+ } else {
+ return this.container.equals(other.container);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append(container).append(outOfLimit)
+ .toHashCode();
+ }
+
+ /**
+ * Check if a container is OPPORTUNISTIC or not. A container is
+ * considered OPPORTUNISTIC only if its execution type is not
+ * null and is OPPORTUNISTIC.
+ */
+ private static boolean isOpportunistic(Container container) {
+ return container.getContainerTokenIdentifier() != null &&
+ ExecutionType.OPPORTUNISTIC.equals(
+ container.getContainerTokenIdentifier().getExecutionType());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5eca1a6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java
index 60c38fe..e239067 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resourc
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
@@ -31,14 +31,12 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext
import org.junit.Test;
import java.io.IOException;
-import java.util.LinkedHashMap;
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_FILE_TASKS;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_PARAM_MEMORY_OOM_CONTROL;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_PARAM_MEMORY_USAGE_BYTES;
-import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -51,13 +49,13 @@ import static org.mockito.Mockito.when;
public class TestDefaultOOMHandler {
/**
- * Test an OOM situation where no containers are running.
+ * Test an OOM situation where there are no containers that can be killed.
*/
@Test(expected = YarnRuntimeException.class)
- public void testNoContainers() throws Exception {
+ public void testExceptionThrownWithNoContainersToKill() throws Exception {
Context context = mock(Context.class);
- when(context.getContainers()).thenReturn(new ConcurrentHashMap<>());
+ when(context.getContainers()).thenReturn(new ConcurrentHashMap<>(0));
CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
when(cGroupsHandler.getCGroupParam(
@@ -66,59 +64,69 @@ public class TestDefaultOOMHandler {
CGROUP_PARAM_MEMORY_OOM_CONTROL))
.thenReturn("under_oom 1").thenReturn("under_oom 0");
- DefaultOOMHandler handler = new DefaultOOMHandler(context, false);
- handler.setCGroupsHandler(cGroupsHandler);
+ DefaultOOMHandler handler = new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
handler.run();
}
/**
- * We have two containers, both out of limit. We should kill the later one.
- *
- * @throws Exception exception
+ * We have two guaranteed containers, both of which are out of limit.
+ * We should kill the later one.
*/
@Test
- public void testBothContainersOOM() throws Exception {
+ public void testBothGuaranteedContainersOverLimitUponOOM() throws Exception {
ConcurrentHashMap<ContainerId, Container> containers =
- new ConcurrentHashMap<>(new LinkedHashMap<>());
+ new ConcurrentHashMap<>();
+ Container c1 = createContainer(1, true, 1L);
+ containers.put(c1.getContainerId(), c1);
+ Container c2 = createContainer(2, true, 2L);
+ containers.put(c2.getContainerId(), c2);
- Container c1 = mock(Container.class);
- ContainerId cid1 = createContainerId(1);
- when(c1.getContainerId()).thenReturn(cid1);
- when(c1.getResource()).thenReturn(Resource.newInstance(10, 1));
- when(c1.getContainerStartTime()).thenReturn((long) 1);
- containers.put(createContainerId(1), c1);
+ ContainerExecutor ex = createContainerExecutor(containers);
+ Context context = mock(Context.class);
+ when(context.getContainers()).thenReturn(containers);
+ when(context.getContainerExecutor()).thenReturn(ex);
- Container c2 = mock(Container.class);
- ContainerId cid2 = createContainerId(2);
- when(c2.getContainerId()).thenReturn(cid2);
- when(c2.getResource()).thenReturn(Resource.newInstance(10, 1));
- when(c2.getContainerStartTime()).thenReturn((long) 2);
- containers.put(cid2, c2);
CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1").thenReturn("under_oom 0");
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid1.toString(), CGROUP_FILE_TASKS))
+ c1.getContainerId().toString(), CGROUP_FILE_TASKS))
.thenReturn("1234").thenReturn("");
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid1.toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
.thenReturn(getMB(11));
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid1.toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
.thenReturn(getMB(11));
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid2.toString(), CGROUP_FILE_TASKS))
+ c2.getContainerId().toString(), CGROUP_FILE_TASKS))
.thenReturn("1235").thenReturn("");
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid2.toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
.thenReturn(getMB(11));
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid2.toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
.thenReturn(getMB(11));
- ContainerExecutor ex = mock(ContainerExecutor.class);
+ DefaultOOMHandler handler =
+ new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
+ handler.run();
- runOOMHandler(containers, cGroupsHandler, ex);
verify(ex, times(1)).signalContainer(
new ContainerSignalContext.Builder()
@@ -131,52 +139,186 @@ public class TestDefaultOOMHandler {
}
/**
- * We have two containers, one out of limit. We should kill that one.
- * This should happen even, if it was started earlier
- *
- * @throws Exception exception
+ * We have two GUARANTEED containers, one of which is out of limit.
+ * We should kill the one that's out of its limit. This should
+ * happen even if it was launched earlier than the other one.
*/
@Test
- public void testOneContainerOOM() throws Exception {
+ public void testOneGuaranteedContainerOverLimitUponOOM() throws Exception {
ConcurrentHashMap<ContainerId, Container> containers =
- new ConcurrentHashMap<>(new LinkedHashMap<>());
+ new ConcurrentHashMap<>();
+ Container c1 = createContainer(1, true, 2L);
+ containers.put(c1.getContainerId(), c1);
+ Container c2 = createContainer(2, true, 1L);
+ containers.put(c2.getContainerId(), c2);
- Container c1 = mock(Container.class);
- ContainerId cid1 = createContainerId(1);
- when(c1.getContainerId()).thenReturn(cid1);
- when(c1.getResource()).thenReturn(Resource.newInstance(10, 1));
- when(c1.getContainerStartTime()).thenReturn((long) 2);
- containers.put(createContainerId(1), c1);
+ ContainerExecutor ex = createContainerExecutor(containers);
+ Context context = mock(Context.class);
+ when(context.getContainers()).thenReturn(containers);
+ when(context.getContainerExecutor()).thenReturn(ex);
- Container c2 = mock(Container.class);
- ContainerId cid2 = createContainerId(2);
- when(c2.getContainerId()).thenReturn(cid2);
- when(c2.getResource()).thenReturn(Resource.newInstance(10, 1));
- when(c2.getContainerStartTime()).thenReturn((long) 1);
- containers.put(cid2, c2);
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1").thenReturn("under_oom 0");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1234").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+
+ // container c2 is out of its limit
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1235").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(11));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(11));
+
+ DefaultOOMHandler handler =
+ new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
+ handler.run();
+
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1235")
+ .setContainer(c2)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(any());
+ }
+
+ /**
+ * We have two GUARANTEE containers, neither of which is out of limit.
+ * We should kill the later launched one.
+ */
+ @Test
+ public void testNoGuaranteedContainerOverLimitOOM() throws Exception {
+ ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>();
+ Container c1 = createContainer(1, true, 1L);
+ containers.put(c1.getContainerId(), c1);
+ Container c2 = createContainer(2, true, 2L);
+ containers.put(c2.getContainerId(), c2);
+
+ ContainerExecutor ex = createContainerExecutor(containers);
+ Context context = mock(Context.class);
+ when(context.getContainers()).thenReturn(containers);
+ when(context.getContainerExecutor()).thenReturn(ex);
CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1").thenReturn("under_oom 0");
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid1.toString(), CGROUP_FILE_TASKS))
+ c1.getContainerId().toString(), CGROUP_FILE_TASKS))
.thenReturn("1234").thenReturn("");
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid1.toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
.thenReturn(getMB(9));
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid1.toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
.thenReturn(getMB(9));
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid2.toString(), CGROUP_FILE_TASKS))
+ c2.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1235").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+
+ DefaultOOMHandler handler =
+ new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
+ handler.run();
+
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1235")
+ .setContainer(c2)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(any());
+ }
+
+ /**
+ * We have two opportunistic containers, both of which are out of limit.
+ * We should kill the later one.
+ */
+ @Test
+ public void testBothOpportunisticContainersOverLimitUponOOM()
+ throws Exception {
+ ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>();
+ Container c1 = createContainer(1, false, 1L);
+ containers.put(c1.getContainerId(), c1);
+ Container c2 = createContainer(2, false, 2L);
+ containers.put(c2.getContainerId(), c2);
+
+ ContainerExecutor ex = createContainerExecutor(containers);
+ Context context = mock(Context.class);
+ when(context.getContainers()).thenReturn(containers);
+ when(context.getContainerExecutor()).thenReturn(ex);
+
+
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1").thenReturn("under_oom 0");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1234").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(11));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(11));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_FILE_TASKS))
.thenReturn("1235").thenReturn("");
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid2.toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
.thenReturn(getMB(11));
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid2.toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
.thenReturn(getMB(11));
- ContainerExecutor ex = mock(ContainerExecutor.class);
- runOOMHandler(containers, cGroupsHandler, ex);
+ DefaultOOMHandler handler =
+ new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
+ handler.run();
+
verify(ex, times(1)).signalContainer(
new ContainerSignalContext.Builder()
@@ -189,51 +331,120 @@ public class TestDefaultOOMHandler {
}
/**
- * We have two containers, neither out of limit. We should kill the later one.
- *
- * @throws Exception exception
+ * We have two OPPORTUNISTIC containers, one of which is out of limit.
+ * We should kill the one that's out of its limit. This should
+ * happen even if it was launched earlier than the other one.
*/
@Test
- public void testNoContainerOOM() throws Exception {
+ public void testOneOpportunisticContainerOverLimitUponOOM() throws Exception {
ConcurrentHashMap<ContainerId, Container> containers =
- new ConcurrentHashMap<>(new LinkedHashMap<>());
+ new ConcurrentHashMap<>();
+ Container c1 = createContainer(1, false, 2L);
+ containers.put(c1.getContainerId(), c1);
+ Container c2 = createContainer(2, false, 1L);
+ containers.put(c2.getContainerId(), c2);
- Container c1 = mock(Container.class);
- ContainerId cid1 = createContainerId(1);
- when(c1.getContainerId()).thenReturn(cid1);
- when(c1.getResource()).thenReturn(Resource.newInstance(10, 1));
- when(c1.getContainerStartTime()).thenReturn((long) 1);
- containers.put(createContainerId(1), c1);
+ ContainerExecutor ex = createContainerExecutor(containers);
+ Context context = mock(Context.class);
+ when(context.getContainers()).thenReturn(containers);
+ when(context.getContainerExecutor()).thenReturn(ex);
+
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1").thenReturn("under_oom 0");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1234").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ // contnainer c2 is out of its limit
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1235").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(11));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(11));
+
+ DefaultOOMHandler handler =
+ new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
+ handler.run();
+
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1235")
+ .setContainer(c2)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(any());
+ }
- Container c2 = mock(Container.class);
- ContainerId cid2 = createContainerId(2);
- when(c2.getContainerId()).thenReturn(cid2);
- when(c2.getResource()).thenReturn(Resource.newInstance(10, 1));
- when(c2.getContainerStartTime()).thenReturn((long) 2);
- containers.put(cid2, c2);
+ /**
+ * We have two OPPORTUNISTIC containers, neither of which is out of limit.
+ * We should kill the later one.
+ */
+ @Test
+ public void testNoOpportunisticContainerOverLimitOOM() throws Exception {
+ ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>();
+ Container c1 = createContainer(1, false, 1L);
+ containers.put(c1.getContainerId(), c1);
+ Container c2 = createContainer(2, false, 2L);
+ containers.put(c2.getContainerId(), c2);
+
+ ContainerExecutor ex = createContainerExecutor(containers);
+ Context context = mock(Context.class);
+ when(context.getContainers()).thenReturn(containers);
+ when(context.getContainerExecutor()).thenReturn(ex);
CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1").thenReturn("under_oom 0");
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid1.toString(), CGROUP_FILE_TASKS))
+ c1.getContainerId().toString(), CGROUP_FILE_TASKS))
.thenReturn("1234").thenReturn("");
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid1.toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
.thenReturn(getMB(9));
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid1.toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
.thenReturn(getMB(9));
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid2.toString(), CGROUP_FILE_TASKS))
+ c2.getContainerId().toString(), CGROUP_FILE_TASKS))
.thenReturn("1235").thenReturn("");
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid2.toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
.thenReturn(getMB(9));
when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
- cid2.toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
.thenReturn(getMB(9));
- ContainerExecutor ex = mock(ContainerExecutor.class);
- runOOMHandler(containers, cGroupsHandler, ex);
+ DefaultOOMHandler handler =
+ new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
+ handler.run();
verify(ex, times(1)).signalContainer(
new ContainerSignalContext.Builder()
@@ -245,43 +456,510 @@ public class TestDefaultOOMHandler {
verify(ex, times(1)).signalContainer(any());
}
- private void runOOMHandler(
- ConcurrentHashMap<ContainerId, Container> containers,
- CGroupsHandler cGroupsHandler, ContainerExecutor ex)
- throws IOException, ResourceHandlerException {
+ /**
+ * We have two OPPORTUNISTIC containers and one GUARANTEED container.
+ * One of the OPPORTUNISTIC container is out of limit.
+ * OOM is resolved after killing the OPPORTUNISTIC container that
+ * exceeded its limit even though it is launched earlier than the
+ * other OPPORTUNISTIC container.
+ */
+ @Test
+ public void testKillOneOverLimitOpportunisticContainerUponOOM()
+ throws Exception {
+ ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>();
+ int currentContainerId = 0;
+ Container c1 = createContainer(currentContainerId++, false, 2);
+ containers.put(c1.getContainerId(), c1);
+ Container c2 = createContainer(currentContainerId++, false, 1);
+ containers.put(c2.getContainerId(), c2);
+ Container c3 = createContainer(currentContainerId++, true, 1);
+ containers.put(c3.getContainerId(), c3);
+
+ ContainerExecutor ex = createContainerExecutor(containers);
Context context = mock(Context.class);
when(context.getContainers()).thenReturn(containers);
+ when(context.getContainerExecutor()).thenReturn(ex);
- when(ex.signalContainer(any()))
- .thenAnswer(invocation -> {
- assertEquals("Wrong pid killed", "1235",
- ((ContainerSignalContext) invocation.getArguments()[0]).getPid());
- return true;
- });
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 0");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1234").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+
+ // container c2 is out of its limit
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1235").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(11));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(11));
+
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1236").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+
+ DefaultOOMHandler handler =
+ new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
+ handler.run();
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1235")
+ .setContainer(c2)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(any());
+ }
+ /**
+ * We have two OPPORTUNISTIC containers and one GUARANTEED container.
+ * None of the containers exceeded its memory limit.
+ * OOM is resolved after killing the most recently launched OPPORTUNISTIC
+ * container.
+ */
+ @Test
+ public void testKillOneLaterOpportunisticContainerUponOOM() throws Exception {
+ ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>();
+ int currentContainerId = 0;
+ Container c1 = createContainer(currentContainerId++, false, 1);
+ containers.put(c1.getContainerId(), c1);
+ Container c2 = createContainer(currentContainerId++, false, 2);
+ containers.put(c2.getContainerId(), c2);
+ Container c3 = createContainer(currentContainerId++, true, 1);
+ containers.put(c3.getContainerId(), c3);
+
+ ContainerExecutor ex = createContainerExecutor(containers);
+ Context context = mock(Context.class);
+ when(context.getContainers()).thenReturn(containers);
+ when(context.getContainerExecutor()).thenReturn(ex);
+
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
when(cGroupsHandler.getCGroupParam(
CGroupsHandler.CGroupController.MEMORY,
"",
CGROUP_PARAM_MEMORY_OOM_CONTROL))
- .thenReturn("under_oom 1").thenReturn("under_oom 0");
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 0");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1234").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1235").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1236").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+
+ DefaultOOMHandler handler =
+ new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
+ handler.run();
+
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1235")
+ .setContainer(c2)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(any());
+ }
+
+ /**
+ * We have two OPPORTUNISTIC containers and one GUARANTEED container.
+ * One of the OPPORTUNISTIC container is out of limit.
+ * OOM is resolved after killing both OPPORTUNISTIC containers.
+ */
+ @Test
+ public void testKillBothOpportunisticContainerUponOOM() throws Exception {
+ int currentContainerId = 0;
+
+ ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>();
+ Container c1 = createContainer(currentContainerId++, false, 2);
+ containers.put(c1.getContainerId(), c1);
+ Container c2 = createContainer(currentContainerId++, false, 1);
+ containers.put(c2.getContainerId(), c2);
+ Container c3 = createContainer(currentContainerId++, true, 1);
+ containers.put(c3.getContainerId(), c3);
+
+ ContainerExecutor ex = createContainerExecutor(containers);
+ Context context = mock(Context.class);
+ when(context.getContainers()).thenReturn(containers);
+ when(context.getContainerExecutor()).thenReturn(ex);
+
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 0");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1234").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1235").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(11));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(11));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1236").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+
+
+ DefaultOOMHandler handler =
+ new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
+ handler.run();
+
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1235")
+ .setContainer(c1)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1234")
+ .setContainer(c2)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(2)).signalContainer(any());
+ }
+
+ /**
+ * We have two OPPORTUNISTIC containers and one GUARANTEED container.
+ * the GUARANTEED container is out of limit. OOM is resolved
+ * after first killing the two OPPORTUNISTIC containers and then the
+ * GUARANTEED container.
+ */
+ @Test
+ public void testKillGuaranteedContainerUponOOM() throws Exception {
+ int currentContainerId = 0;
+
+ ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>();
+ Container c1 = createContainer(currentContainerId++, false, 2);
+ containers.put(c1.getContainerId(), c1);
+ Container c2 = createContainer(currentContainerId++, false, 1);
+ containers.put(c2.getContainerId(), c2);
+ Container c3 = createContainer(currentContainerId++, true, 1);
+ containers.put(c3.getContainerId(), c3);
+
+ ContainerExecutor ex = createContainerExecutor(containers);
+ Context context = mock(Context.class);
+ when(context.getContainers()).thenReturn(containers);
+ when(context.getContainerExecutor()).thenReturn(ex);
+
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 0");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1234").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1235").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1236").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(11));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(11));
+
+ DefaultOOMHandler handler =
+ new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
+ handler.run();
+
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1234")
+ .setContainer(c1)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1235")
+ .setContainer(c1)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1236")
+ .setContainer(c1)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(3)).signalContainer(any());
+ }
+
+ /**
+ * We have two OPPORTUNISTIC containers and one GUARANTEED container.
+ * None of the containers exceeded its memory limit.
+ * OOM is resolved after killing all running containers.
+ */
+ @Test
+ public void testKillAllContainersUponOOM() throws Exception {
+ int currentContainerId = 0;
+ ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>();
+ Container c1 = createContainer(currentContainerId++, false, 1);
+ containers.put(c1.getContainerId(), c1);
+ Container c2 = createContainer(currentContainerId++, false, 2);
+ containers.put(c2.getContainerId(), c2);
+ Container c3 = createContainer(currentContainerId++, true, 1);
+ containers.put(c3.getContainerId(), c3);
+
+ ContainerExecutor ex = createContainerExecutor(containers);
+ Context context = mock(Context.class);
+ when(context.getContainers()).thenReturn(containers);
when(context.getContainerExecutor()).thenReturn(ex);
- DefaultOOMHandler handler = new DefaultOOMHandler(context, false);
- handler.setCGroupsHandler(cGroupsHandler);
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 0");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1234").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1235").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1236").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ DefaultOOMHandler handler =
+ new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
handler.run();
+
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1234")
+ .setContainer(c2)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1235")
+ .setContainer(c1)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1236")
+ .setContainer(c3)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(3)).signalContainer(any());
}
- private class AppId extends ApplicationIdPBImpl {
- AppId(long clusterTs, int appId) {
- this.setClusterTimestamp(clusterTs);
- this.setId(appId);
- }
+ /**
+ * We have two OPPORTUNISTIC containers and one GUARANTEED container.
+ * None of the containers exceeded its memory limit.
+ * OOM is not resolved even after killing all running containers.
+ * A YarnRuntimeException is excepted to be thrown.
+ */
+ @Test(expected = YarnRuntimeException.class)
+ public void testOOMUnresolvedAfterKillingAllContainers() throws Exception {
+ int currentContainerId = 0;
+
+ ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>();
+ Container c1 = createContainer(currentContainerId++, false, 1);
+ containers.put(c1.getContainerId(), c1);
+ Container c2 = createContainer(currentContainerId++, false, 2);
+ containers.put(c2.getContainerId(), c2);
+ Container c3 = createContainer(currentContainerId++, true, 3);
+ containers.put(c3.getContainerId(), c3);
+
+ ContainerExecutor ex = createContainerExecutor(containers);
+ Context context = mock(Context.class);
+ when(context.getContainers()).thenReturn(containers);
+ when(context.getContainerExecutor()).thenReturn(ex);
+
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 1")
+ .thenReturn("under_oom 1");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1234").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1235").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1236").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+
+ DefaultOOMHandler handler =
+ new DefaultOOMHandler(context, false) {
+ @Override
+ protected CGroupsHandler getCGroupsHandler() {
+ return cGroupsHandler;
+ }
+ };
+ handler.run();
}
- private ContainerId createContainerId(int id) {
- ApplicationId applicationId = new AppId(1, 1);
+ private static ContainerId createContainerId(int id) {
+ ApplicationId applicationId = ApplicationId.newInstance(1, 1);
ApplicationAttemptId applicationAttemptId
= mock(ApplicationAttemptId.class);
@@ -295,13 +973,41 @@ public class TestDefaultOOMHandler {
return containerId;
}
- ContainerTokenIdentifier getToken() {
- ContainerTokenIdentifier id = mock(ContainerTokenIdentifier.class);
- when(id.getVersion()).thenReturn(1);
- return id;
+ private static Container createContainer(int containerId,
+ boolean guaranteed, long launchTime) {
+ Container c1 = mock(Container.class);
+ ContainerId cid1 = createContainerId(containerId);
+ when(c1.getContainerId()).thenReturn(cid1);
+
+ ContainerTokenIdentifier token = mock(ContainerTokenIdentifier.class);
+ ExecutionType type =
+ guaranteed ? ExecutionType.GUARANTEED : ExecutionType.OPPORTUNISTIC;
+ when(token.getExecutionType()).thenReturn(type);
+ when(c1.getContainerTokenIdentifier()).thenReturn(token);
+
+ when(c1.getResource()).thenReturn(Resource.newInstance(10, 1));
+ when(c1.getContainerLaunchTime()).thenReturn(launchTime);
+
+ return c1;
}
String getMB(long mb) {
return Long.toString(mb * 1024 * 1024);
}
+
+ private static ContainerExecutor createContainerExecutor(
+ ConcurrentHashMap<ContainerId, Container> containers)
+ throws IOException {
+ ContainerExecutor ex = mock(ContainerExecutor.class);
+ when(ex.signalContainer(any())).thenAnswer(
+ invocation -> {
+ Object[] arguments = invocation.getArguments();
+ Container container = ((ContainerSignalContext)
+ arguments[0]).getContainer();
+ // remove container from NM context immediately
+ containers.remove(container.getContainerId());
+ return true;
+ });
+ return ex;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5eca1a6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index 77ebd34..325709b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -242,6 +242,11 @@ public class MockContainer implements Container {
}
@Override
+ public long getContainerLaunchTime() {
+ return 0;
+ }
+
+ @Override
public ResourceMappings getResourceMappings() {
return null;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/16] hadoop git commit: HADOOP-15482. Upgrade jackson-databind to
version 2.9.5. Contributed by Lokesh Jain.
Posted by su...@apache.org.
HADOOP-15482. Upgrade jackson-databind to version 2.9.5. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c42dcc7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c42dcc7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c42dcc7c
Branch: refs/heads/HDFS-12943
Commit: c42dcc7c47340d517563890269c6c112996e8897
Parents: 3b88fe2
Author: Jitendra Pandey <ji...@apache.org>
Authored: Thu Jun 7 23:00:26 2018 -0700
Committer: Jitendra Pandey <ji...@apache.org>
Committed: Thu Jun 7 23:00:26 2018 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c42dcc7c/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 8edfd76..8cb5bfc 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -69,7 +69,7 @@
<!-- jackson versions -->
<jackson.version>1.9.13</jackson.version>
- <jackson2.version>2.9.4</jackson2.version>
+ <jackson2.version>2.9.5</jackson2.version>
<!-- SLF4J version -->
<slf4j.version>1.7.25</slf4j.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/16] hadoop git commit: HDDS-127. Add CloseContainerEventHandler
in SCM. Contributed by Shashikant Banerjee.
Posted by su...@apache.org.
HDDS-127. Add CloseContainerEventHandler in SCM.
Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78761e87
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78761e87
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78761e87
Branch: refs/heads/HDFS-12943
Commit: 78761e87a7f3012ef2d96e294d55b323b76b7c42
Parents: ba303b1
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Jun 7 14:35:22 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Jun 7 14:35:22 2018 -0700
----------------------------------------------------------------------
.../container/CloseContainerEventHandler.java | 83 +++++++++
.../hdds/scm/container/ContainerMapping.java | 5 +
.../scm/container/ContainerStateManager.java | 9 +
.../hadoop/hdds/scm/container/Mapping.java | 6 +
.../TestCloseContainerEventHandler.java | 177 +++++++++++++++++++
5 files changed, 280 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78761e87/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
new file mode 100644
index 0000000..bc95b55
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * <p>http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * <p>Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.server.events.TypedEvent;
+import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * In case of a node failure, volume failure, volume out of spapce, node
+ * out of space etc, CLOSE_CONTAINER_EVENT will be triggered.
+ * CloseContainerEventHandler is the handler for CLOSE_CONTAINER_EVENT.
+ * When a close container event is fired, a close command for the container
+ * should be sent to all the datanodes in the pipeline and containerStateManager
+ * needs to update the container state to Closing.
+ */
+public class CloseContainerEventHandler implements EventHandler<ContainerID> {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(CloseContainerEventHandler.class);
+
+ public static final TypedEvent<ContainerID> CLOSE_CONTAINER_EVENT =
+ new TypedEvent<>(ContainerID.class);
+
+ private final Mapping containerManager;
+
+ public CloseContainerEventHandler(Mapping containerManager) {
+ this.containerManager = containerManager;
+ }
+
+ @Override
+ public void onMessage(ContainerID containerID, EventPublisher publisher) {
+
+ LOG.info("Close container Event triggered for container : {}",
+ containerID.getId());
+ ContainerStateManager stateManager = containerManager.getStateManager();
+ ContainerInfo info = stateManager.getContainer(containerID);
+ if (info == null) {
+ LOG.info("Container with id : {} does not exist", containerID.getId());
+ return;
+ }
+ if (info.getState() == HddsProtos.LifeCycleState.OPEN) {
+ for (DatanodeDetails datanode : info.getPipeline().getMachines()) {
+ containerManager.getNodeManager().addDatanodeCommand(datanode.getUuid(),
+ new CloseContainerCommand(containerID.getId()));
+ }
+ try {
+ // Finalize event will make sure the state of the container transitions
+ // from OPEN to CLOSING in containerStateManager.
+ stateManager
+ .updateContainerState(info, HddsProtos.LifeCycleEvent.FINALIZE);
+ } catch (SCMException ex) {
+ LOG.error("Failed to update the container state for container : {}"
+ + containerID);
+ }
+ } else {
+ LOG.info("container with id : {} is in {} state and need not be closed.",
+ containerID.getId(), info.getState());
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78761e87/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index f5fe46a..b961c38 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -588,6 +588,11 @@ public class ContainerMapping implements Mapping {
}
}
+ @Override
+ public NodeManager getNodeManager() {
+ return nodeManager;
+ }
+
@VisibleForTesting
public MetadataStore getContainerStore() {
return containerStore;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78761e87/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
index 4895b78..9dfa660 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
@@ -445,6 +445,15 @@ public class ContainerStateManager implements Closeable {
factor, type);
}
+ /**
+ * Returns the containerInfo for the given container id.
+ * @param containerID id of the container
+ * @return ContainerInfo containerInfo
+ * @throws IOException
+ */
+ public ContainerInfo getContainer(ContainerID containerID) {
+ return containers.getContainerInfo(containerID.getId());
+ }
@Override
public void close() throws IOException {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78761e87/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
index ee8e344..ab42520 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
import java.io.Closeable;
import java.io.IOException;
@@ -103,4 +104,9 @@ public interface Mapping extends Closeable {
ContainerReportsProto reports)
throws IOException;
+ /**
+ * Returns the nodeManager.
+ * @return NodeManager
+ */
+ NodeManager getNodeManager();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78761e87/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
new file mode 100644
index 0000000..09ade3e
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.CREATE;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.CREATED;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB;
+
+/**
+ * Tests the closeContainerEventHandler class.
+ */
+public class TestCloseContainerEventHandler {
+
+ private static Configuration configuration;
+ private static MockNodeManager nodeManager;
+ private static ContainerMapping mapping;
+ private static long size;
+ private static File testDir;
+ private static EventQueue eventQueue;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ configuration = SCMTestUtils.getConf();
+ size = configuration
+ .getLong(OZONE_SCM_CONTAINER_SIZE_GB, OZONE_SCM_CONTAINER_SIZE_DEFAULT)
+ * 1024 * 1024 * 1024;
+ testDir = GenericTestUtils
+ .getTestDir(TestCloseContainerEventHandler.class.getSimpleName());
+ configuration
+ .set(OzoneConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
+ nodeManager = new MockNodeManager(true, 10);
+ mapping = new ContainerMapping(configuration, nodeManager, 128);
+ eventQueue = new EventQueue();
+ eventQueue.addHandler(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT,
+ new CloseContainerEventHandler(mapping));
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ if (mapping != null) {
+ mapping.close();
+ }
+ FileUtil.fullyDelete(testDir);
+ }
+
+ @Test
+ public void testIfCloseContainerEventHadnlerInvoked() {
+ GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
+ .captureLogs(CloseContainerEventHandler.LOG);
+ eventQueue.fireEvent(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT,
+ new ContainerID(Math.abs(new Random().nextLong())));
+ eventQueue.processAll(1000);
+ Assert.assertTrue(logCapturer.getOutput()
+ .contains("Close container Event triggered for container"));
+ }
+
+ @Test
+ public void testCloseContainerEventWithInvalidContainer() {
+ long id = Math.abs(new Random().nextLong());
+ GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
+ .captureLogs(CloseContainerEventHandler.LOG);
+ eventQueue.fireEvent(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT,
+ new ContainerID(id));
+ eventQueue.processAll(1000);
+ Assert.assertTrue(logCapturer.getOutput()
+ .contains("Container with id : " + id + " does not exist"));
+ }
+
+ @Test
+ public void testCloseContainerEventWithValidContainers() throws IOException {
+
+ GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
+ .captureLogs(CloseContainerEventHandler.LOG);
+ ContainerInfo info = mapping
+ .allocateContainer(HddsProtos.ReplicationType.STAND_ALONE,
+ HddsProtos.ReplicationFactor.ONE, "ozone");
+ ContainerID id = new ContainerID(info.getContainerID());
+ DatanodeDetails datanode = info.getPipeline().getLeader();
+ int closeCount = nodeManager.getCommandCount(datanode);
+ eventQueue.fireEvent(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT, id);
+ eventQueue.processAll(1000);
+ // At this point of time, the allocated container is not in open
+ // state, so firing close container event should not queue CLOSE
+ // command in the Datanode
+ Assert.assertEquals(0, nodeManager.getCommandCount(datanode));
+ // Make sure the information is logged
+ Assert.assertTrue(logCapturer.getOutput().contains(
+ "container with id : " + id.getId()
+ + " is in ALLOCATED state and need not be closed"));
+ //Execute these state transitions so that we can close the container.
+ mapping.updateContainerState(id.getId(), CREATE);
+ mapping.updateContainerState(id.getId(), CREATED);
+ eventQueue.fireEvent(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT,
+ new ContainerID(info.getContainerID()));
+ eventQueue.processAll(1000);
+ Assert.assertEquals(closeCount + 1, nodeManager.getCommandCount(datanode));
+ Assert.assertEquals(HddsProtos.LifeCycleState.CLOSING,
+ mapping.getStateManager().getContainer(id).getState());
+ }
+
+ @Test
+ public void testCloseContainerEventWithRatis() throws IOException {
+
+ GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
+ .captureLogs(CloseContainerEventHandler.LOG);
+ ContainerInfo info = mapping
+ .allocateContainer(HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE, "ozone");
+ ContainerID id = new ContainerID(info.getContainerID());
+ int[] closeCount = new int[3];
+ eventQueue.fireEvent(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT, id);
+ eventQueue.processAll(1000);
+ int i = 0;
+ for (DatanodeDetails details : info.getPipeline().getMachines()) {
+ closeCount[i] = nodeManager.getCommandCount(details);
+ i++;
+ }
+ i = 0;
+ for (DatanodeDetails details : info.getPipeline().getMachines()) {
+ Assert.assertEquals(closeCount[i], nodeManager.getCommandCount(details));
+ i++;
+ }
+ // Make sure the information is logged
+ Assert.assertTrue(logCapturer.getOutput().contains(
+ "container with id : " + id.getId()
+ + " is in ALLOCATED state and need not be closed"));
+ //Execute these state transitions so that we can close the container.
+ mapping.updateContainerState(id.getId(), CREATE);
+ mapping.updateContainerState(id.getId(), CREATED);
+ eventQueue.fireEvent(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT,
+ new ContainerID(info.getContainerID()));
+ eventQueue.processAll(1000);
+ i = 0;
+ // Make sure close is queued for each datanode on the pipeline
+ for (DatanodeDetails details : info.getPipeline().getMachines()) {
+ Assert.assertEquals(closeCount[i] + 1,
+ nodeManager.getCommandCount(details));
+ Assert.assertEquals(HddsProtos.LifeCycleState.CLOSING,
+ mapping.getStateManager().getContainer(id).getState());
+ i++;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/16] hadoop git commit: YARN-8359. Exclude containermanager.linux
test classes on Windows. Contributed by Jason Lowe.
Posted by su...@apache.org.
YARN-8359. Exclude containermanager.linux test classes on Windows. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b88fe25
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b88fe25
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b88fe25
Branch: refs/heads/HDFS-12943
Commit: 3b88fe25baf130cd7a77590f9ded5b0bf028ef75
Parents: 351cf87
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Jun 7 17:09:31 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Jun 7 17:13:03 2018 -0700
----------------------------------------------------------------------
.../hadoop-yarn-server-nodemanager/pom.xml | 21 ++++++++++++++++++++
1 file changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b88fe25/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index 5146820..26a5220 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -294,6 +294,27 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>native-win</id>
+ <activation>
+ <os>
+ <family>Windows</family>
+ </os>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.**</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
<build>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/16] hadoop git commit: HDFS-13664. Refactor
ConfiguredFailoverProxyProvider to make inheritance easier. Contributed by
Chao Sun.
Posted by su...@apache.org.
HDFS-13664. Refactor ConfiguredFailoverProxyProvider to make inheritance easier. Contributed by Chao Sun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fba1c42a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fba1c42a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fba1c42a
Branch: refs/heads/HDFS-12943
Commit: fba1c42adc1c8ae57951e1865ec2ab05c8707bdf
Parents: cf41083
Author: Chao Sun <su...@apache.org>
Authored: Fri Jun 8 16:36:42 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Fri Jun 8 16:36:42 2018 -0700
----------------------------------------------------------------------
.../namenode/ha/ConfiguredFailoverProxyProvider.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fba1c42a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
index e9c8791..58f4943 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
@@ -52,11 +52,11 @@ public class ConfiguredFailoverProxyProvider<T> extends
protected final Configuration conf;
protected final List<AddressRpcProxyPair<T>> proxies =
new ArrayList<AddressRpcProxyPair<T>>();
- private final UserGroupInformation ugi;
+ protected final UserGroupInformation ugi;
protected final Class<T> xface;
private int currentProxyIndex = 0;
- private final HAProxyFactory<T> factory;
+ protected final HAProxyFactory<T> factory;
public ConfiguredFailoverProxyProvider(Configuration conf, URI uri,
Class<T> xface, HAProxyFactory<T> factory) {
@@ -122,6 +122,10 @@ public class ConfiguredFailoverProxyProvider<T> extends
@Override
public synchronized ProxyInfo<T> getProxy() {
AddressRpcProxyPair<T> current = proxies.get(currentProxyIndex);
+ return getProxy(current);
+ }
+
+ protected ProxyInfo<T> getProxy(AddressRpcProxyPair<T> current) {
if (current.namenode == null) {
try {
current.namenode = factory.createProxy(conf,
@@ -147,7 +151,7 @@ public class ConfiguredFailoverProxyProvider<T> extends
* A little pair object to store the address and connected RPC proxy object to
* an NN. Note that {@link AddressRpcProxyPair#namenode} may be null.
*/
- private static class AddressRpcProxyPair<T> {
+ protected static class AddressRpcProxyPair<T> {
public final InetSocketAddress address;
public T namenode;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org