You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by mm...@apache.org on 2022/06/08 10:08:27 UTC
[ignite] branch master updated: IGNITE-17120 Move ignite-yarn to the Ignite Extensions project (#10074)
This is an automated email from the ASF dual-hosted git repository.
mmuzaf pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 747e5cb4320 IGNITE-17120 Move ignite-yarn to the Ignite Extensions project (#10074)
747e5cb4320 is described below
commit 747e5cb4320019fed48605174a740a70f3b35c84
Author: Maxim Muzafarov <ma...@gmail.com>
AuthorDate: Wed Jun 8 13:08:17 2022 +0300
IGNITE-17120 Move ignite-yarn to the Ignite Extensions project (#10074)
---
DEVNOTES.txt | 10 -
assembly/dependencies-apache-ignite-slim.xml | 1 -
assembly/libs/README.txt | 1 -
.../ignite-for-spark/installation.adoc | 2 +-
modules/yarn/README.txt | 28 -
modules/yarn/licenses/apache-2.0.txt | 202 ------
modules/yarn/pom.xml | 167 -----
.../org/apache/ignite/yarn/ApplicationMaster.java | 453 --------------
.../org/apache/ignite/yarn/ClusterProperties.java | 489 ---------------
.../org/apache/ignite/yarn/IgniteContainer.java | 86 ---
.../org/apache/ignite/yarn/IgniteProvider.java | 280 ---------
.../org/apache/ignite/yarn/IgniteYarnClient.java | 216 -------
.../java/org/apache/ignite/yarn/package-info.java | 23 -
.../apache/ignite/yarn/utils/IgniteYarnUtils.java | 106 ----
.../org/apache/ignite/yarn/utils/package-info.java | 23 -
.../src/main/resources/ignite-default-config.xml | 35 --
modules/yarn/src/main/resources/ignite.properties | 23 -
.../org/apache/ignite/IgniteYarnTestSuite.java | 32 -
.../yarn/IgniteApplicationMasterSelfTest.java | 683 ---------------------
parent-internal/pom.xml | 4 -
pom.xml | 1 -
21 files changed, 1 insertion(+), 2864 deletions(-)
diff --git a/DEVNOTES.txt b/DEVNOTES.txt
index e51704d68cb..fb9b2e1ca93 100644
--- a/DEVNOTES.txt
+++ b/DEVNOTES.txt
@@ -205,16 +205,6 @@ To test compliance with JCache TCK use:
./mvnw test -P-release,jcache-tck -pl :ignite-core -am
-
-Ignite Yarn Maven Build Instructions
-====================================
-
- cd ./modules/yarn
- mvn clean package
-
-Look for ignite-yarn-<version>.jar in ./target directory.
-
-
Run tests
=========
To run tests locally use:
diff --git a/assembly/dependencies-apache-ignite-slim.xml b/assembly/dependencies-apache-ignite-slim.xml
index df6d80d173b..d6bc9e19d14 100644
--- a/assembly/dependencies-apache-ignite-slim.xml
+++ b/assembly/dependencies-apache-ignite-slim.xml
@@ -154,7 +154,6 @@
<exclude>${project.groupId}:ignite-osgi-paxlogging</exclude>
<exclude>${project.groupId}:ignite-ssh</exclude>
<exclude>${project.groupId}:ignite-web</exclude>
- <exclude>${project.groupId}:ignite-yarn</exclude>
<exclude>${project.groupId}:ignite-zookeeper</exclude>
</excludes>
<sources>
diff --git a/assembly/libs/README.txt b/assembly/libs/README.txt
index e210d3a30e8..fa0aeea1047 100644
--- a/assembly/libs/README.txt
+++ b/assembly/libs/README.txt
@@ -88,7 +88,6 @@ The following modules are available:
- ignite-ssh (for starting grid nodes on remote machines)
- ignite-urideploy (for URI-based deployment)
- ignite-web (for Web Sessions Clustering)
-- ignite-yarn (for integration with Apache Hadoop Yarn)
- ignite-zookeeper (for cluster discovery based on Apache Zookeeper)
For example, if you want to use Apache Ignite Spring-based configuration,
diff --git a/docs/_docs/extensions-and-integrations/ignite-for-spark/installation.adoc b/docs/_docs/extensions-and-integrations/ignite-for-spark/installation.adoc
index 3d2586f8d0e..05cdf643cfa 100644
--- a/docs/_docs/extensions-and-integrations/ignite-for-spark/installation.adoc
+++ b/docs/_docs/extensions-and-integrations/ignite-for-spark/installation.adoc
@@ -120,7 +120,7 @@ The following shows how to fill in `spark.driver.extraClassPath` parameters:
[source, shell]
----
-spark.executor.extraClassPath /opt/ignite/libs/*:/opt/ignite/libs/optional/ignite-spark-ext/*:/opt/ignite/libs/optional/ignite-log4j/*:/opt/ignite/libs/optional/ignite-yarn/*:/opt/ignite/libs/ignite-spring/*
+spark.executor.extraClassPath /opt/ignite/libs/*:/opt/ignite/libs/optional/ignite-spark-ext/*:/opt/ignite/libs/optional/ignite-log4j/*:/opt/ignite/libs/optional/ignite-yarn-ext/*:/opt/ignite/libs/ignite-spring/*
----
=== Source Code Configuration
diff --git a/modules/yarn/README.txt b/modules/yarn/README.txt
deleted file mode 100644
index 5cdd4a2fe95..00000000000
--- a/modules/yarn/README.txt
+++ /dev/null
@@ -1,28 +0,0 @@
-Apache Ignite Yarn Module
-------------------------
-
-Apache Ignite Yarn module provides integration Apache Ignite with Apache Hadoop Yarn.
-
-Importing Apache Ignite Yarn Module In Maven Project
--------------------------------------
-
-If you are using Maven to manage dependencies of your project, you can add Cloud module
-dependency like this (replace '${ignite.version}' with actual Ignite version you are
-interested in):
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
- http://maven.apache.org/xsd/maven-4.0.0.xsd">
- ...
- <dependencies>
- ...
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-yarn</artifactId>
- <version>${ignite.version}</version>
- </dependency>
- ...
- </dependencies>
- ...
-</project>
diff --git a/modules/yarn/licenses/apache-2.0.txt b/modules/yarn/licenses/apache-2.0.txt
deleted file mode 100644
index d6456956733..00000000000
--- a/modules/yarn/licenses/apache-2.0.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/modules/yarn/pom.xml b/modules/yarn/pom.xml
deleted file mode 100644
index 22d78c02e7c..00000000000
--- a/modules/yarn/pom.xml
+++ /dev/null
@@ -1,167 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--
- POM file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-parent-internal</artifactId>
- <version>${revision}</version>
- <relativePath>../../parent-internal/pom.xml</relativePath>
- </parent>
-
- <artifactId>ignite-yarn</artifactId>
- <url>http://ignite.apache.org</url>
-
- <properties>
- <hadoop-yarn.version>3.3.1</hadoop-yarn.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <version>${hadoop-yarn.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop-yarn.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- <version>${commons.collections.version}</version>
- </dependency>
-
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- <version>${commons.codec.version}</version>
- </dependency>
-
- <dependency>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- <version>${commons.beanutils.version}</version>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>ignite-tools</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.12</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <!-- Copy ignite.properties from core module since it is not available as dependency. -->
- <artifactId>maven-resources-plugin</artifactId>
- <version>3.1.0</version>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>validate</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>src/main/resources</outputDirectory>
- <resources>
- <resource>
- <directory>../core/src/main/resources</directory>
- <includes>
- <include>ignite.properties</include>
- </includes>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>2.4.1</version>
- <configuration>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- <archive>
- <manifest>
- <mainClass>org.apache.ignite.yarn.IgniteYarnClient</mainClass>
- </manifest>
- </archive>
- <appendAssemblyId>false</appendAssemblyId>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-deploy-plugin</artifactId>
- <version>2.8.2</version>
- <configuration>
- <skip>false</skip>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
deleted file mode 100644
index 886f2d75743..00000000000
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
+++ /dev/null
@@ -1,453 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yarn;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.NMClient;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.ignite.yarn.utils.IgniteYarnUtils;
-
-/**
- * Application master request containers from Yarn and decides how many resources will be occupied.
- */
-public class ApplicationMaster implements AMRMClientAsync.CallbackHandler {
- /** */
- public static final Logger log = Logger.getLogger(ApplicationMaster.class.getSimpleName());
-
- /** Default port range. */
- public static final String DEFAULT_PORT = ":47500..47510";
-
- /** Delimiter char. */
- public static final String DELIM = ",";
-
- /** */
- private long schedulerTimeout = TimeUnit.SECONDS.toMillis(1);
-
- /** Yarn configuration. */
- private final YarnConfiguration conf;
-
- /** Cluster properties. */
- private final ClusterProperties props;
-
- /** Network manager. */
- private NMClient nmClient;
-
- /** Resource manager. */
- private AMRMClientAsync<AMRMClient.ContainerRequest> rmClient;
-
- /** Ignite path. */
- private final Path ignitePath;
-
- /** Config path. */
- private Path cfgPath;
-
- /** Hadoop file system. */
- private FileSystem fs;
-
- /** Buffered tokens to be injected into newly allocated containers. */
- private ByteBuffer allTokens;
-
- /** Running containers. */
- private final Map<ContainerId, IgniteContainer> containers = new ConcurrentHashMap<>();
-
- /**
- * @param ignitePath Hdfs path to ignite.
- * @param props Cluster properties.
- */
- public ApplicationMaster(String ignitePath, ClusterProperties props) throws Exception {
- this.conf = new YarnConfiguration();
- this.props = props;
- this.ignitePath = new Path(ignitePath);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void onContainersAllocated(List<Container> conts) {
- for (Container c : conts) {
- if (checkContainer(c)) {
- log.log(Level.INFO, "Container {0} allocated", c.getId());
-
- try {
- ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
-
- if (UserGroupInformation.isSecurityEnabled())
- // Set the tokens to the newly allocated container:
- ctx.setTokens(allTokens.duplicate());
-
- Map<String, String> env = new HashMap<>(ctx.getEnvironment());
-
- Map<String, String> systemEnv = System.getenv();
-
- for (String key : systemEnv.keySet()) {
- if (key.matches("^IGNITE_[_0-9A-Z]+$"))
- env.put(key, systemEnv.get(key));
- }
-
- env.put("IGNITE_TCP_DISCOVERY_ADDRESSES", getAddress(c.getNodeId().getHost()));
-
- if (props.jvmOpts() != null && !props.jvmOpts().isEmpty())
- env.put("JVM_OPTS", props.jvmOpts());
-
- ctx.setEnvironment(env);
-
- Map<String, LocalResource> resources = new HashMap<>();
-
- resources.put("ignite", IgniteYarnUtils.setupFile(ignitePath, fs, LocalResourceType.ARCHIVE));
- resources.put("ignite-config.xml", IgniteYarnUtils.setupFile(cfgPath, fs, LocalResourceType.FILE));
-
- if (props.licencePath() != null)
- resources.put("gridgain-license.xml",
- IgniteYarnUtils.setupFile(new Path(props.licencePath()), fs, LocalResourceType.FILE));
-
- if (props.userLibs() != null)
- resources.put("libs", IgniteYarnUtils.setupFile(new Path(props.userLibs()), fs,
- LocalResourceType.FILE));
-
- ctx.setLocalResources(resources);
-
- ctx.setCommands(
- Collections.singletonList(
- (props.licencePath() != null ? "cp gridgain-license.xml ./ignite/*/ || true && " : "")
- + "cp -r ./libs/* ./ignite/*/libs/ || true && "
- + "./ignite/*/bin/ignite.sh "
- + "./ignite-config.xml"
- + " -J-Xmx" + ((int)props.memoryPerNode()) + "m"
- + " -J-Xms" + ((int)props.memoryPerNode()) + "m"
- + IgniteYarnUtils.YARN_LOG_OUT
- ));
-
- log.log(Level.INFO, "Launching container: {0}.", c.getId());
-
- nmClient.startContainer(c, ctx);
-
- containers.put(c.getId(),
- new IgniteContainer(
- c.getId(),
- c.getNodeId(),
- c.getResource().getVirtualCores(),
- c.getResource().getMemory()));
- }
- catch (Exception ex) {
- log.log(Level.WARNING, "Error launching container " + c.getId(), ex);
- }
- }
- else {
- log.log(Level.WARNING, "Container {0} check failed. Releasing...", c.getId());
-
- rmClient.releaseAssignedContainer(c.getId());
- }
- }
- }
-
- /**
- * @param cont Container.
- * @return {@code True} if container satisfies requirements.
- */
- private boolean checkContainer(Container cont) {
- // Check limit on running nodes.
- if (props.instances() <= containers.size()) {
- log.log(Level.WARNING, "Limit on running nodes exceeded. ({0} of {1} max)",
- new Object[] {containers.size(), props.instances()});
-
- return false;
- }
-
- // Check host name
- if (props.hostnameConstraint() != null
- && props.hostnameConstraint().matcher(cont.getNodeId().getHost()).matches()) {
- log.log(Level.WARNING, "Wrong host name '{0}'. It didn't match to '{1}' pattern.",
- new Object[] {cont.getNodeId().getHost(), props.hostnameConstraint().toString()});
-
- return false;
- }
-
- // Check that slave satisfies min requirements.
- if (cont.getResource().getVirtualCores() < props.cpusPerNode()
- || cont.getResource().getMemory() < props.totalMemoryPerNode()) {
- log.log(Level.WARNING, "Container resources not sufficient requirements. Host: {0}, cpu: {1}, mem: {2}",
- new Object[] {cont.getNodeId().getHost(), cont.getResource().getVirtualCores(),
- cont.getResource().getMemory()});
-
- return false;
- }
-
- return true;
- }
-
- /**
- * @return Address running nodes.
- */
- private String getAddress(String addr) {
- if (containers.isEmpty()) {
- if (addr != null && !addr.isEmpty())
- return addr + DEFAULT_PORT;
-
- return "";
- }
-
- StringBuilder sb = new StringBuilder();
-
- for (IgniteContainer cont : containers.values())
- sb.append(cont.nodeId.getHost()).append(DEFAULT_PORT).append(DELIM);
-
- return sb.substring(0, sb.length() - 1);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void onContainersCompleted(List<ContainerStatus> statuses) {
- for (ContainerStatus status : statuses) {
- containers.remove(status.getContainerId());
-
- log.log(Level.INFO, "Container completed. Container id: {0}. State: {1}.",
- new Object[]{status.getContainerId(), status.getState()});
- }
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void onNodesUpdated(List<NodeReport> updated) {
- for (NodeReport node : updated) {
- // If node unusable.
- if (node.getNodeState().isUnusable()) {
- for (IgniteContainer cont : containers.values()) {
- if (cont.nodeId().equals(node.getNodeId())) {
- containers.remove(cont.id());
-
- log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.",
- new Object[]{node.getNodeId().getHost(), node.getNodeState()});
- }
- }
-
- log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.",
- new Object[]{node.getNodeId().getHost(), node.getNodeState()});
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public void onShutdownRequest() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void onError(Throwable t) {
- nmClient.stop();
- }
-
- /** {@inheritDoc} */
- @Override public float getProgress() {
- return 50;
- }
-
- /**
- * @param args Args.
- * @throws Exception If failed.
- */
- public static void main(String[] args) throws Exception {
- ClusterProperties props = ClusterProperties.from();
-
- ApplicationMaster master = new ApplicationMaster(args[0], props);
-
- master.init();
-
- master.run();
- }
-
- /**
- * Runs application master.
- *
- * @throws Exception If failed.
- */
- public void run() throws Exception {
- // Register with ResourceManager
- rmClient.registerApplicationMaster("", 0, "");
-
- log.log(Level.INFO, "Application master registered.");
-
- // Priority for worker containers - priorities are intra-application
- Priority priority = Records.newRecord(Priority.class);
- priority.setPriority(0);
-
- try {
- // Check ignite cluster.
- while (!nmClient.isInState(Service.STATE.STOPPED)) {
- int runningCnt = containers.size();
-
- if (runningCnt < props.instances() && checkAvailableResource()) {
- // Resource requirements for worker containers.
- Resource capability = Records.newRecord(Resource.class);
-
- capability.setMemory((int)props.totalMemoryPerNode());
- capability.setVirtualCores((int)props.cpusPerNode());
-
- for (int i = 0; i < props.instances() - runningCnt; ++i) {
- // Make container requests to ResourceManager
- AMRMClient.ContainerRequest containerAsk =
- new AMRMClient.ContainerRequest(capability, null, null, priority);
-
- rmClient.addContainerRequest(containerAsk);
-
- log.log(Level.INFO, "Making request. Memory: {0}, cpu {1}.",
- new Object[]{props.totalMemoryPerNode(), props.cpusPerNode()});
- }
- }
-
- TimeUnit.MILLISECONDS.sleep(schedulerTimeout);
- }
- }
- catch (InterruptedException ignored) {
- // Un-register with ResourceManager
- rmClient.unregisterApplicationMaster(FinalApplicationStatus.KILLED, "", "");
-
- log.log(Level.WARNING, "Application master killed.");
- }
- catch (Exception e) {
- // Un-register with ResourceManager
- rmClient.unregisterApplicationMaster(FinalApplicationStatus.FAILED, "", "");
-
- log.log(Level.SEVERE, "Application master failed.", e);
- }
- }
-
- /**
- * @return {@code True} if cluster contains available resources.
- */
- private boolean checkAvailableResource() {
- Resource availableRes = rmClient.getAvailableResources();
-
- return availableRes == null || availableRes.getMemory() >= props.totalMemoryPerNode()
- && availableRes.getVirtualCores() >= props.cpusPerNode();
- }
-
- /**
- * @throws IOException
- */
- public void init() throws IOException {
- if (UserGroupInformation.isSecurityEnabled()) {
- Credentials cred = UserGroupInformation.getCurrentUser().getCredentials();
-
- allTokens = IgniteYarnUtils.createTokenBuffer(cred);
- }
-
- fs = FileSystem.get(conf);
-
- nmClient = NMClient.createNMClient();
-
- nmClient.init(conf);
- nmClient.start();
-
- // Create async application master.
- rmClient = AMRMClientAsync.createAMRMClientAsync(300, this);
-
- rmClient.init(conf);
-
- rmClient.start();
-
- if (props.igniteCfg() == null || props.igniteCfg().isEmpty()) {
- InputStream input = Thread.currentThread().getContextClassLoader()
- .getResourceAsStream(IgniteYarnUtils.DEFAULT_IGNITE_CONFIG);
-
- cfgPath = new Path(props.igniteWorkDir() + File.separator + IgniteYarnUtils.DEFAULT_IGNITE_CONFIG);
-
- // Create file. Override by default.
- FSDataOutputStream outputStream = fs.create(cfgPath, true);
-
- IOUtils.copy(input, outputStream);
-
- IOUtils.closeQuietly(input);
-
- IOUtils.closeQuietly(outputStream);
- }
- else
- cfgPath = new Path(props.igniteCfg());
- }
-
- /**
- * Sets NMClient.
- *
- * @param nmClient NMClient.
- */
- public void setNmClient(NMClient nmClient) {
- this.nmClient = nmClient;
- }
-
- /**
- * Sets RMClient
- *
- * @param rmClient AMRMClientAsync.
- */
- public void setRmClient(AMRMClientAsync<AMRMClient.ContainerRequest> rmClient) {
- this.rmClient = rmClient;
- }
-
- /**
- * Sets scheduler timeout.
- *
- * @param schedulerTimeout Scheduler timeout.
- */
- public void setSchedulerTimeout(long schedulerTimeout) {
- this.schedulerTimeout = schedulerTimeout;
- }
-
- /**
- * Sets file system.
- * @param fs File system.
- */
- public void setFs(FileSystem fs) {
- this.fs = fs;
- }
-
- /**
- * JUST FOR TESTING!!!
- *
- * @return Running containers.
- */
- @Deprecated
- public Map<ContainerId, IgniteContainer> getContainers() {
- return containers;
- }
-}
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
deleted file mode 100644
index 35f5da22c5b..00000000000
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
+++ /dev/null
@@ -1,489 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yarn;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
-
-/**
- * Cluster settings.
- */
-public class ClusterProperties {
- /** */
- private static final Logger log = Logger.getLogger(ClusterProperties.class.getSimpleName());
-
- /** */
- public static final String EMPTY_STRING = "";
-
- /** */
- public static final String IGNITE_CLUSTER_NAME = "IGNITE_CLUSTER_NAME";
-
- /** */
- public static final String DEFAULT_CLUSTER_NAME = "ignite-cluster";
-
- /** */
- public static final double DEFAULT_CPU_PER_NODE = 2;
-
- /** */
- public static final double DEFAULT_MEM_PER_NODE = 2048;
-
- /**
- * The minimum memory overhead: overhead is by default 0.1* MEMORY_PER_NODE,
- * with a minimum of DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE.
- */
- public static final double DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE = 384;
-
- /** Cluster name. */
- private String clusterName = DEFAULT_CLUSTER_NAME;
-
- /** */
- public static final String IGNITE_RUN_CPU_PER_NODE = "IGNITE_RUN_CPU_PER_NODE";
-
- /** CPU limit. */
- private double cpuPerNode = DEFAULT_CPU_PER_NODE;
-
- /** */
- public static final String IGNITE_MEMORY_PER_NODE = "IGNITE_MEMORY_PER_NODE";
-
- /** Memory limit. */
- private double memPerNode = DEFAULT_MEM_PER_NODE;
-
- /** */
- public static final String IGNITE_MEMORY_OVERHEAD_PER_NODE = "IGNITE_MEMORY_OVERHEAD_PER_NODE";
-
- /** Memory over head to request yarn. */
- private double memOverHeadPerNode = 0;
-
- /** */
- public static final String IGNITE_NODE_COUNT = "IGNITE_NODE_COUNT";
-
- /** */
- public static final double DEFAULT_IGNITE_NODE_COUNT = 3;
-
- /** Node count limit. */
- private double nodeCnt = DEFAULT_IGNITE_NODE_COUNT;
-
- /** */
- public static final String IGNITE_URL = "IGNITE_URL";
-
- /** Ignite version. */
- private String igniteUrl = null;
-
- /** */
- public static final String IGNITE_WORKING_DIR = "IGNITE_WORKING_DIR";
-
- /** */
- public static final String DEFAULT_IGNITE_WORK_DIR = "/ignite/workdir/";
-
- /** Ignite work directory. */
- private String igniteWorkDir = DEFAULT_IGNITE_WORK_DIR;
-
- /** */
- public static final String IGNITE_PATH = "IGNITE_PATH";
-
- /** Ignite path. */
- private String ignitePath = null;
-
- /** */
- public static final String LICENCE_PATH = "LICENCE_PATH";
-
- /** Licence path. */
- private String licencePath = null;
-
- /** */
- public static final String IGNITE_JVM_OPTS = "IGNITE_JVM_OPTS";
-
- /** JVM opts. */
- private String jvmOpts = null;
-
- /** */
- public static final String IGNITE_LOCAL_WORK_DIR = "IGNITE_LOCAL_WORK_DIR";
-
- /** */
- public static final String DEFAULT_IGNITE_LOCAL_WORK_DIR = "./ignite-releases/";
-
- /** Ignite local work directory. */
- private String igniteLocalWorkDir = DEFAULT_IGNITE_LOCAL_WORK_DIR;
-
- /** */
- public static final String IGNITE_RELEASES_DIR = "IGNITE_RELEASES_DIR";
-
- /** */
- public static final String DEFAULT_IGNITE_RELEASES_DIR = "/ignite/releases/";
-
- /** Ignite local work directory. */
- private String igniteReleasesDir = DEFAULT_IGNITE_RELEASES_DIR;
-
- /** */
- public static final String IGNITE_USERS_LIBS = "IGNITE_USERS_LIBS";
-
- /** Path to users libs. */
- private String userLibs = null;
-
- /** */
- public static final String IGNITE_CONFIG_XML = "IGNITE_XML_CONFIG";
-
- /** Ignite config. */
- private String igniteCfg = null;
-
- /** */
- public static final String IGNITE_HOSTNAME_CONSTRAINT = "IGNITE_HOSTNAME_CONSTRAINT";
-
- /** Url to ignite config. */
- private Pattern hostnameConstraint = null;
-
- /** Ignite Yarn Queue */
- public static final String IGNITE_YARN_QUEUE = "IGNITE_YARN_QUEUE";
-
- /** Ignite Yarn default Queue */
- public static final String DEFAULT_IGNITE_YARN_QUEUE = "default";
-
- /** Path to users libs. */
- private String yarnQueue = DEFAULT_IGNITE_YARN_QUEUE;
-
- /** */
- public ClusterProperties() {
- // No-op.
- }
-
- /**
- * @return Cluster name.
- */
- public String clusterName() {
- return clusterName;
- }
-
- /**
- * @return CPU count limit.
- */
- public double cpusPerNode() {
- return cpuPerNode;
- }
-
- /**
- * Sets CPU count limit.
- *
- * @param cpu CPU per node count limit.
- */
- public void cpusPerNode(double cpu) {
- this.cpuPerNode = cpu;
- }
-
- /**
- * @return mem limit.
- */
- public double memoryPerNode() {
- return memPerNode;
- }
-
- /**
- * Sets mem limit.
- *
- * @param mem Memory.
- */
- public void memoryPerNode(double mem) {
- this.memPerNode = mem;
- }
-
- /**
- * @return Memory overhead for requested memory.
- */
- public double memoryOverHeadPerNode() {
- return memOverHeadPerNode;
- }
-
- /**
- * Sets memory overhead requested to YARN.
- *
- * @param memOverHeadPerNode Memory over head per node.
- */
- public void memoryOverHeadPerNode(double memOverHeadPerNode) {
- this.memOverHeadPerNode = memOverHeadPerNode;
- }
-
- /**
- * @return Provide the total memory requested to ResourceManagers (memoryPerNode + memoryOverheadPerNode).
- */
- public double totalMemoryPerNode() {
- return memoryPerNode() + memoryOverHeadPerNode();
- }
-
- /**
- * @return Instance count limit.
- */
- public double instances() {
- return nodeCnt;
- }
-
- /**
- * Sets instance count limit.
- *
- * @param nodeCnt Node instance count limit.
- */
- public void instances(int nodeCnt) {
- this.nodeCnt = nodeCnt;
- }
-
- /**
- * Sets hostname constraint.
- *
- * @param pattern Hostname pattern.
- */
- public void hostnameConstraint(Pattern pattern) {
- this.hostnameConstraint = pattern;
- }
-
- /**
- * Sets Yarn Queue
- *
- * @param queue queue name.
- */
- public void yarnQueue(String queue) {
- this.yarnQueue = queue;
- }
-
- /**
- * @return Ignite version.
- */
- public String igniteUrl() {
- return igniteUrl;
- }
-
- /**
- * @return Working directory.
- */
- public String igniteWorkDir() {
- return igniteWorkDir;
- }
-
- /**
- * @return Local working directory.
- */
- public String igniteLocalWorkDir() {
- return igniteLocalWorkDir;
- }
-
- /**
- * @return Ignite releases dir.
- */
- public String igniteReleasesDir() {
- return igniteReleasesDir;
- }
-
- /**
- * @return User's libs.
- */
- public String userLibs() {
- return userLibs;
- }
-
- /**
- * @return Ignite configuration.
- */
- public String igniteCfg() {
- return igniteCfg;
- }
-
- /**
- * @return Licence path.
- */
- public String licencePath() {
- return licencePath;
- }
-
- /**
- * @return Ignite hdfs path.
- */
- public String ignitePath() {
- return ignitePath;
- }
-
- /**
- * @return Jvm opts.
- */
- public String jvmOpts() {
- return jvmOpts;
- }
-
- /**
- * @return Host name constraint.
- */
- public Pattern hostnameConstraint() {
- return hostnameConstraint;
- }
-
- /**
- * @return Yarn Queue
- */
- public String yarnQueue() {
- return yarnQueue;
- }
-
- /**
- * Instantiate a ClusterProperties from a set of properties.
- *
- * @param props If {@code null} will be used system properties.
- * @return Cluster properties.
- */
- private static ClusterProperties fromProperties(Properties props) {
- ClusterProperties prop = new ClusterProperties();
-
- prop.clusterName = getStringProperty(IGNITE_CLUSTER_NAME, props, DEFAULT_CLUSTER_NAME);
-
- prop.cpuPerNode = getDoubleProperty(IGNITE_RUN_CPU_PER_NODE, props, DEFAULT_CPU_PER_NODE);
- prop.memPerNode = getDoubleProperty(IGNITE_MEMORY_PER_NODE, props, DEFAULT_MEM_PER_NODE);
- // The minimum memory overhead: overhead is by default 0.1* MEMORY_PER_NODE,
- // with a minimum of DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE
- prop.memOverHeadPerNode = getDoubleProperty(IGNITE_MEMORY_OVERHEAD_PER_NODE, props,
- Math.max( 0.1 * prop.memPerNode, DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE));
- prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, props, DEFAULT_IGNITE_NODE_COUNT);
-
- prop.igniteUrl = getStringProperty(IGNITE_URL, props, null);
- prop.ignitePath = getStringProperty(IGNITE_PATH, props, null);
- prop.licencePath = getStringProperty(LICENCE_PATH, props, null);
- prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, props, null);
- prop.igniteWorkDir = getStringProperty(IGNITE_WORKING_DIR, props, DEFAULT_IGNITE_WORK_DIR);
- prop.igniteLocalWorkDir = getStringProperty(IGNITE_LOCAL_WORK_DIR, props, DEFAULT_IGNITE_LOCAL_WORK_DIR);
- prop.igniteReleasesDir = getStringProperty(IGNITE_RELEASES_DIR, props, DEFAULT_IGNITE_RELEASES_DIR);
- prop.igniteCfg = getStringProperty(IGNITE_CONFIG_XML, props, null);
- prop.userLibs = getStringProperty(IGNITE_USERS_LIBS, props, null);
- prop.yarnQueue = getStringProperty(IGNITE_YARN_QUEUE, props, DEFAULT_IGNITE_YARN_QUEUE);
-
- String pattern = getStringProperty(IGNITE_HOSTNAME_CONSTRAINT, props, null);
-
- if (pattern != null) {
- try {
- prop.hostnameConstraint = Pattern.compile(pattern);
- }
- catch (PatternSyntaxException e) {
- log.log(Level.WARNING, "IGNITE_HOSTNAME_CONSTRAINT has invalid pattern. It will be ignore.", e);
- }
- }
-
- return prop;
- }
-
- /**
- * @param config Path to config file.
- * @return Cluster configuration.
- */
- public static ClusterProperties from(String config) {
- try {
- Properties props = null;
-
- if (config != null) {
- props = new Properties();
-
- props.load(new FileInputStream(config));
- }
-
- return fromProperties(props);
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * @return Cluster configuration.
- */
- public static ClusterProperties from() {
- return fromProperties(null);
- }
-
- /**
- * Convert to properties to map.
- *
- * @return Key-value map.
- */
- public Map<String, String> toEnvs() {
- Map<String, String> envs = new HashMap<>();
-
- envs.put(IGNITE_CLUSTER_NAME, toEnvVal(clusterName));
-
- envs.put(IGNITE_RUN_CPU_PER_NODE, toEnvVal(cpuPerNode));
- envs.put(IGNITE_MEMORY_PER_NODE, toEnvVal(memPerNode));
- envs.put(IGNITE_MEMORY_OVERHEAD_PER_NODE, toEnvVal(memOverHeadPerNode));
- envs.put(IGNITE_NODE_COUNT, toEnvVal(nodeCnt));
-
- envs.put(IGNITE_URL, toEnvVal(igniteUrl));
- envs.put(IGNITE_PATH, toEnvVal(ignitePath));
- envs.put(LICENCE_PATH, toEnvVal(licencePath));
- envs.put(IGNITE_JVM_OPTS, toEnvVal(jvmOpts));
- envs.put(IGNITE_WORKING_DIR, toEnvVal(igniteWorkDir));
- envs.put(IGNITE_LOCAL_WORK_DIR, toEnvVal(igniteLocalWorkDir));
- envs.put(IGNITE_RELEASES_DIR, toEnvVal(igniteReleasesDir));
- envs.put(IGNITE_CONFIG_XML, toEnvVal(igniteCfg));
- envs.put(IGNITE_USERS_LIBS, toEnvVal(userLibs));
- envs.put(IGNITE_YARN_QUEUE, toEnvVal(yarnQueue));
-
- if (hostnameConstraint != null)
- envs.put(IGNITE_HOSTNAME_CONSTRAINT, toEnvVal(hostnameConstraint.pattern()));
-
- return envs;
- }
-
- /**
- * @param name Property name.
- * @param fileProps Property file.
- * @return Property value.
- */
- private static double getDoubleProperty(String name, Properties fileProps, Double defaultVal) {
- if (fileProps != null && fileProps.containsKey(name))
- return Double.valueOf(fileProps.getProperty(name));
-
- String property = System.getProperty(name);
-
- if (property == null)
- property = System.getenv(name);
-
- return property == null || property.isEmpty() ? defaultVal : Double.valueOf(property);
- }
-
- /**
- * @param name Property name.
- * @param fileProps Property file.
- * @return Property value.
- */
- private static String getStringProperty(String name, Properties fileProps, String defaultVal) {
- if (fileProps != null && fileProps.containsKey(name))
- return fileProps.getProperty(name);
-
- String property = System.getProperty(name);
-
- if (property == null)
- property = System.getenv(name);
-
- return property == null || property.isEmpty() ? defaultVal : property;
- }
-
- /**
- * @param val Value.
- * @return If val is null {@code EMPTY_STRING} else to string.
- */
- private String toEnvVal(Object val) {
- return val == null ? EMPTY_STRING : val.toString();
- }
-}
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteContainer.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteContainer.java
deleted file mode 100644
index 1e76f0ea4e7..00000000000
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteContainer.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yarn;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-
-/**
- * Information about launched task.
- */
-public class IgniteContainer {
- /** */
- public final ContainerId id;
-
- /** */
- public final NodeId nodeId;
-
- /** */
- public final double cpuCores;
-
- /** */
- public final double mem;
-
- /**
- * Ignite launched task.
- *
- * @param id Container ID.
- * @param nodeId Node id.
- * @param cpuCores Cpu cores count.
- * @param mem Memory
- */
- public IgniteContainer(ContainerId id, NodeId nodeId, double cpuCores, double mem) {
- this.id = id;
- this.nodeId = nodeId;
- this.cpuCores = cpuCores;
- this.mem = mem;
- }
-
- /**
- * @return Id.
- */
- public ContainerId id() {
- return id;
- }
-
- /**
- * @return Host.
- */
- public NodeId nodeId() {
- return nodeId;
- }
-
- /**
- * @return Cores count.
- */
- public double cpuCores() {
- return cpuCores;
- }
-
- /**
- * @return Memory.
- */
- public double mem() {
- return mem;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "IgniteTask [host=" + nodeId.getHost() + ", cpuCores=" + cpuCores + ", mem=" + mem + ']';
- }
-}
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java
deleted file mode 100644
index 311058cfee8..00000000000
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yarn;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.nio.channels.Channels;
-import java.util.Properties;
-import java.util.logging.Logger;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.ignite.yarn.utils.IgniteYarnUtils;
-
-/**
- * Downloads and stores Ignite release.
- */
-public class IgniteProvider {
- /** */
- public static final Logger log = Logger.getLogger(IgniteProvider.class.getSimpleName());
-
- /** */
- public static final String DOWNLOAD_LINK = "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/ignite/";
-
- /** */
- private ClusterProperties props;
-
- /** */
- private FileSystem fs;
-
- /**
- * @param props Cluster properties.
- * @param fs Hadoop file system.
- */
- public IgniteProvider(ClusterProperties props, FileSystem fs) {
- this.props = props;
- this.fs = fs;
- }
-
- /**
- * @return Path to current Ignite release.
- */
- public Path getIgnite() throws Exception {
- File folder = checkDownloadFolder();
-
- Properties verProps = new Properties();
-
- try (InputStream is = IgniteProvider.class.getClassLoader().getResourceAsStream("ignite.properties")) {
- verProps.load(is);
- }
-
- String curVer = verProps.getProperty("ignite.version");
-
- if (curVer == null || curVer.isEmpty())
- throw new IllegalStateException("Failed to determine Ignite version");
-
- log.info("Searching for Ignite release " + curVer);
-
- File release = findIgnite(folder, curVer);
-
- if (release == null) {
- Path releaseOnHdfs = findIgnite(fs, props.igniteReleasesDir(), curVer);
-
- if (releaseOnHdfs != null)
- return releaseOnHdfs;
-
- release = updateIgnite(curVer);
- }
-
- return IgniteYarnUtils.copyLocalToHdfs(fs, release.getAbsolutePath(),
- props.igniteReleasesDir() + File.separator + release.getName());
- }
-
- /**
- * @param folder Folder.
- * @param curVer Ignite version.
- * @return Ignite archives.
- */
- private File findIgnite(File folder, String curVer) {
- String[] files = folder.list();
-
- if (files != null) {
- for (String fileName : files) {
- if (fileName.equals(igniteRelease(curVer))) {
- log.info("Found local release at " + folder.getAbsolutePath());
-
- return new File(folder, fileName);
- }
- }
- }
-
- return null;
- }
-
- /**
- * @param fs File system,
- * @param folder Folder.
- * @param curVer Ignite version.
- * @return Ignite archives.
- */
- private Path findIgnite(FileSystem fs, String folder, String curVer) {
- FileStatus[] fileStatuses = null;
-
- try {
- fileStatuses = fs.listStatus(new Path(folder));
- }
- catch (FileNotFoundException ignored) {
- // Ignore. Folder doesn't exist.
- }
- catch (Exception e) {
- throw new RuntimeException("Couldn't get list files from hdfs.", e);
- }
-
- if (fileStatuses != null) {
- for (FileStatus file : fileStatuses) {
- String fileName = file.getPath().getName();
-
- if (fileName.equals(igniteRelease(curVer))) {
- log.info("Found HDFS release at " + file.getPath());
-
- return file.getPath();
- }
- }
- }
-
- return null;
- }
-
- /**
- * @param igniteUrl Download link.
- * @return Ignite.
- */
- public Path getIgnite(String igniteUrl) throws Exception {
- checkDownloadFolder();
-
- // Download ignite.
- String fileName = downloadIgnite(igniteUrl);
-
- Path dst = new Path(props.igniteReleasesDir() + File.separator + fileName);
-
- log.info("Using specified release at " + igniteUrl);
-
- if (!fs.exists(dst))
- fs.copyFromLocalFile(new Path(props.igniteLocalWorkDir() + File.separator + fileName), dst);
-
- return dst;
- }
-
- /**
- * @return File name.
- */
- private static String igniteRelease(String version) {
- return "apache-ignite-" + version + "-bin.zip";
- }
-
- /**
- * @param curVer Ignite version.
- * @return Current version if the current version is latest; new ignite version otherwise.
- */
- private File updateIgnite(String curVer) {
- try {
- // Such as https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/ignite/2.7.0/apache-ignite-2.7.0-bin.zip
- URL url = new URL(DOWNLOAD_LINK + curVer + "/" + igniteRelease(curVer));
-
- HttpURLConnection conn = (HttpURLConnection)url.openConnection();
-
- int code = conn.getResponseCode();
-
- String redirectUrl = conn.getURL().toString();
-
- if (code == 301 || code == 302) {
- redirectUrl = conn.getHeaderField("Location");
-
- conn.disconnect();
-
- conn = (HttpURLConnection)new URL(redirectUrl).openConnection();
- }
- else if (code != 200)
- throw new RuntimeException("Got unexpected response code. Response code: " + code);
-
- checkDownloadFolder();
-
- File ignite = new File(props.igniteLocalWorkDir(), fileName(redirectUrl));
-
- FileOutputStream outFile = new FileOutputStream(ignite);
-
- outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE);
-
- outFile.close();
-
- log.info("Found remote release at " + redirectUrl);
-
- return ignite;
- }
- catch (IOException e) {
- throw new RuntimeException("Failed update ignite.", e);
- }
- }
-
- /**
- * @param igniteUrl Url to ignite.
- * @return Ignite file name.
- */
- private String downloadIgnite(String igniteUrl) {
- try {
- URL url = new URL(igniteUrl);
-
- HttpURLConnection conn = (HttpURLConnection)url.openConnection();
-
- int code = conn.getResponseCode();
-
- if (code == 200) {
- String fileName = fileName(url.toString());
-
- String filePath = props.igniteLocalWorkDir() + File.separator + fileName;
-
- if (new File(filePath).exists())
- return fileName;
-
- FileOutputStream outFile = new FileOutputStream(filePath);
-
- outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE);
-
- outFile.close();
-
- return fileName;
- }
- else
- throw new RuntimeException("Got unexpected response code. Response code: " + code);
- }
- catch (IOException e) {
- throw new RuntimeException("Failed update ignite.", e);
- }
- }
-
- /**
- * @return Download folder.
- */
- private File checkDownloadFolder() {
- File file = new File(props.igniteLocalWorkDir());
-
- if (!file.exists())
- file.mkdirs();
-
- if (!file.exists())
- throw new RuntimeException("Couldn't create local directory! Path: " + file.toURI());
-
- return file;
- }
-
- /**
- * @param url URL.
- * @return File name.
- */
- private static String fileName(String url) {
- String[] split = url.split("/");
-
- return split[split.length - 1];
- }
-}
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
deleted file mode 100644
index f9753e87283..00000000000
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yarn;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.ignite.yarn.utils.IgniteYarnUtils;
-
-import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-
-/**
- * Ignite yarn client.
- */
-public class IgniteYarnClient {
- /** */
- public static final Logger log = Logger.getLogger(IgniteYarnClient.class.getSimpleName());
-
- /**
- * Main methods has one mandatory parameter and one optional parameter.
- *
- * @param args Path to jar mandatory parameter and property file is optional.
- */
- public static void main(String[] args) throws Exception {
- checkArguments(args);
-
- // Set path to app master jar.
- String pathAppMasterJar = args[0];
-
- ClusterProperties props = ClusterProperties.from(args.length == 2 ? args[1] : null);
-
- YarnConfiguration conf = new YarnConfiguration();
- YarnClient yarnClient = YarnClient.createYarnClient();
- yarnClient.init(conf);
- yarnClient.start();
-
- // Create application via yarnClient
- YarnClientApplication app = yarnClient.createApplication();
-
- FileSystem fs = FileSystem.get(conf);
-
- Path ignite;
-
- // Load ignite and jar
- if (props.ignitePath() == null)
- ignite = getIgnite(props, fs);
- else
- ignite = new Path(props.ignitePath());
-
- // Upload the jar file to HDFS.
- Path appJar = IgniteYarnUtils.copyLocalToHdfs(fs, pathAppMasterJar,
- props.igniteWorkDir() + File.separator + IgniteYarnUtils.JAR_NAME);
-
- // Set up the container launch context for the application master
- ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
-
- amContainer.setCommands(
- Collections.singletonList(
- Environment.JAVA_HOME.$() + "/bin/java -Xmx512m " + ApplicationMaster.class.getName()
- + IgniteYarnUtils.SPACE + ignite.toUri()
- + IgniteYarnUtils.YARN_LOG_OUT
- )
- );
-
- // Setup jar for ApplicationMaster
- LocalResource appMasterJar = IgniteYarnUtils.setupFile(appJar, fs, LocalResourceType.FILE);
-
- amContainer.setLocalResources(Collections.singletonMap(IgniteYarnUtils.JAR_NAME, appMasterJar));
-
- // Setup CLASSPATH for ApplicationMaster
- Map<String, String> appMasterEnv = props.toEnvs();
-
- setupAppMasterEnv(appMasterEnv, conf);
-
- amContainer.setEnvironment(appMasterEnv);
-
- // Setup security tokens
- if (UserGroupInformation.isSecurityEnabled()) {
- Credentials creds = new Credentials();
-
- String tokRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
-
- if (tokRenewer == null || tokRenewer.isEmpty())
- throw new IOException("Master Kerberos principal for the RM is not set.");
-
- log.info("Found RM principal: " + tokRenewer);
-
- final Token<?> tokens[] = fs.addDelegationTokens(tokRenewer, creds);
-
- if (tokens != null)
- log.info("File system delegation tokens: " + Arrays.toString(tokens));
-
- amContainer.setTokens(IgniteYarnUtils.createTokenBuffer(creds));
- }
-
- // Set up resource type requirements for ApplicationMaster
- Resource capability = Records.newRecord(Resource.class);
- capability.setMemory(512);
- capability.setVirtualCores(1);
-
- // Finally, set-up ApplicationSubmissionContext for the application
- ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
- appContext.setApplicationName("ignition"); // application name
- appContext.setAMContainerSpec(amContainer);
- appContext.setResource(capability);
- appContext.setQueue(props.yarnQueue()); // queue
-
- // Submit application
- ApplicationId appId = appContext.getApplicationId();
-
- yarnClient.submitApplication(appContext);
-
- log.log(Level.INFO, "Submitted application. Application id: {0}", appId);
-
- ApplicationReport appReport = yarnClient.getApplicationReport(appId);
- YarnApplicationState appState = appReport.getYarnApplicationState();
-
- while (appState == YarnApplicationState.NEW ||
- appState == YarnApplicationState.NEW_SAVING ||
- appState == YarnApplicationState.SUBMITTED ||
- appState == YarnApplicationState.ACCEPTED) {
- TimeUnit.SECONDS.sleep(1L);
-
- appReport = yarnClient.getApplicationReport(appId);
-
- if (appState != YarnApplicationState.ACCEPTED
- && appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED)
- log.log(Level.INFO, "Application {0} is ACCEPTED.", appId);
-
- appState = appReport.getYarnApplicationState();
- }
-
- log.log(Level.INFO, "Application {0} is {1}.", new Object[]{appId, appState});
- }
-
- /**
- * Check input arguments.
- *
- * @param args Arguments.
- */
- private static void checkArguments(String[] args) {
- if (args.length < 1)
- throw new IllegalArgumentException("Invalid arguments.");
- }
-
- /**
- * @param props Properties.
- * @param fileSystem Hdfs file system.
- * @return Hdfs path to ignite node.
- * @throws Exception
- */
- private static Path getIgnite(ClusterProperties props, FileSystem fileSystem) throws Exception {
- IgniteProvider provider = new IgniteProvider(props, fileSystem);
-
- String igniteUrl = props.igniteUrl();
-
- if (igniteUrl != null)
- return provider.getIgnite(igniteUrl);
-
- return provider.getIgnite();
- }
-
- /**
- * @param envs Environment variables.
- * @param conf Yarn configuration.
- */
- private static void setupAppMasterEnv(Map<String, String> envs, YarnConfiguration conf) {
- for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH))
- Apps.addToEnvironment(envs, Environment.CLASSPATH.name(),
- c.trim(), File.pathSeparator);
-
- Apps.addToEnvironment(envs,
- Environment.CLASSPATH.name(),
- Environment.PWD.$() + File.separator + "*",
- File.pathSeparator);
- }
-}
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/package-info.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/package-info.java
deleted file mode 100644
index 06c8f52a198..00000000000
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains classes to support integration with Apache Hadoop Yarn.
- */
-
-package org.apache.ignite.yarn;
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java
deleted file mode 100644
index 345d032da8e..00000000000
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yarn.utils;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
-
-import static org.apache.hadoop.yarn.api.ApplicationConstants.LOG_DIR_EXPANSION_VAR;
-
-/**
- * Utils.
- */
-public class IgniteYarnUtils {
- /** */
- public static final String DEFAULT_IGNITE_CONFIG = "ignite-default-config.xml";
-
- /** */
- public static final String SPACE = " ";
-
- /** */
- public static final String JAR_NAME = "ignite-yarn.jar";
-
- /** */
- public static final String YARN_LOG_OUT =
- " 1>" + LOG_DIR_EXPANSION_VAR + "/stdout" +
- " 2>" + LOG_DIR_EXPANSION_VAR + "/stderr";
-
- /**
- * @param file Path.
- * @param fs File system.
- * @param type Local resource type.
- * @return Resource file.
- * @throws Exception If failed.
- */
- public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type)
- throws Exception {
- LocalResource resource = Records.newRecord(LocalResource.class);
-
- file = fs.makeQualified(file);
-
- FileStatus stat = fs.getFileStatus(file);
-
- resource.setResource(ConverterUtils.getYarnUrlFromPath(file));
- resource.setSize(stat.getLen());
- resource.setTimestamp(stat.getModificationTime());
- resource.setType(type);
- resource.setVisibility(LocalResourceVisibility.APPLICATION);
-
- return resource;
- }
-
- /**
- * @param fs File system.
- * @param src Source path.
- * @param dst Destination path.
- * @return Path to file to hdfs file system.
- */
- public static Path copyLocalToHdfs(FileSystem fs, String src, String dst) throws Exception {
- Path dstPath = new Path(dst);
-
- // Local file isn't removed, dst file override.
- fs.copyFromLocalFile(false, true, new Path(src), dstPath);
-
- return dstPath;
- }
-
- /**
- * Creates a ByteBuffer with serialized {@link Credentials}.
- *
- * @param creds The credentials.
- * @return The ByteBuffer with the credentials.
- * @throws IOException
- */
- public static ByteBuffer createTokenBuffer(Credentials creds) throws IOException {
- DataOutputBuffer dob = new DataOutputBuffer();
-
- creds.writeTokenStorageToStream(dob);
-
- return ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- }
-}
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/package-info.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/package-info.java
deleted file mode 100644
index 30bdf7dd7f3..00000000000
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Utility and helper classes.
- */
-
-package org.apache.ignite.yarn.utils;
diff --git a/modules/yarn/src/main/resources/ignite-default-config.xml b/modules/yarn/src/main/resources/ignite-default-config.xml
deleted file mode 100644
index 2f263987f49..00000000000
--- a/modules/yarn/src/main/resources/ignite-default-config.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<beans xmlns="http://www.springframework.org/schema/beans"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://www.springframework.org/schema/beans
- http://www.springframework.org/schema/beans/spring-beans.xsd">
- <bean class="org.apache.ignite.configuration.IgniteConfiguration">
- <property name="discoverySpi">
- <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
- <property name="ipFinder">
- <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"/>
- </property>
-
- <property name="joinTimeout" value="60000"/>
- </bean>
- </property>
- </bean>
-</beans>
diff --git a/modules/yarn/src/main/resources/ignite.properties b/modules/yarn/src/main/resources/ignite.properties
deleted file mode 100644
index 3653f5f33f2..00000000000
--- a/modules/yarn/src/main/resources/ignite.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-ignite.version=2.14.0-SNAPSHOT
-ignite.build=0
-ignite.revision=DEV
-ignite.rel.date=01011970
-ignite.update.status.params=test=vfvfvskfkeievskjv
-ignite.update.notifier.enabled.by.default=false
diff --git a/modules/yarn/src/test/java/org/apache/ignite/IgniteYarnTestSuite.java b/modules/yarn/src/test/java/org/apache/ignite/IgniteYarnTestSuite.java
deleted file mode 100644
index 9bc5c2802a7..00000000000
--- a/modules/yarn/src/test/java/org/apache/ignite/IgniteYarnTestSuite.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite;
-
-import org.apache.ignite.yarn.IgniteApplicationMasterSelfTest;
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
-/**
- * Apache Hadoop Yarn integration tests.
- */
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- IgniteApplicationMasterSelfTest.class
-})
-public class IgniteYarnTestSuite {
-}
diff --git a/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java b/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
deleted file mode 100644
index 7be154fd5df..00000000000
--- a/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
+++ /dev/null
@@ -1,683 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yarn;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.regex.Pattern;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.NMClient;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Application master tests.
- */
-public class IgniteApplicationMasterSelfTest {
- /** */
- private ApplicationMaster appMaster;
-
- /** */
- private ClusterProperties props;
-
- /** */
- private RMMock rmMock = new RMMock();
-
- /**
- * @throws Exception If failed.
- */
- @Before
- public void setUp() throws Exception {
- props = new ClusterProperties();
- appMaster = new ApplicationMaster("test", props);
-
- appMaster.setSchedulerTimeout(10000);
-
- rmMock.clear();
- }
-
- /**
- * @throws Exception If failed.
- */
- @Test
- public void testContainerAllocate() throws Exception {
- appMaster.setRmClient(rmMock);
- appMaster.setNmClient(new NMMock());
-
- props.cpusPerNode(2);
- props.memoryPerNode(1024);
- props.instances(3);
-
- Thread thread = runAppMaster(appMaster);
-
- List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 2, 1000);
-
- interruptedThread(thread);
-
- assertEquals(3, contRequests.size());
-
- for (AMRMClient.ContainerRequest req : contRequests) {
- assertEquals(2, req.getCapability().getVirtualCores());
- assertEquals(1024, req.getCapability().getMemory());
- }
- }
-
- /**
- * Tests whether memory overhead is allocated within container memory.
- *
- * @throws Exception If failed.
- */
- @Test
- public void testMemoryOverHeadAllocation() throws Exception {
- appMaster.setRmClient(rmMock);
- appMaster.setNmClient(new NMMock());
-
- props.cpusPerNode(2);
- props.memoryPerNode(1024);
- props.memoryOverHeadPerNode(512);
- props.instances(3);
-
- Thread thread = runAppMaster(appMaster);
-
- List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1, 1000);
-
- interruptedThread(thread);
-
- assertEquals(3, contRequests.size());
-
- for (AMRMClient.ContainerRequest req : contRequests) {
- assertEquals(2, req.getCapability().getVirtualCores());
- assertEquals(1024 + 512, req.getCapability().getMemory());
- }
- }
-
- /**
- * Tests whether memory overhead prevents from allocating container.
- *
- * @throws Exception If failed.
- */
- @Test
- public void testMemoryOverHeadPreventAllocation() throws Exception {
- rmMock.availableRes(new MockResource(1024, 2));
- appMaster.setRmClient(rmMock);
- appMaster.setNmClient(new NMMock());
-
- props.cpusPerNode(2);
- props.memoryPerNode(1024);
- props.memoryOverHeadPerNode(512);
- props.instances(3);
-
- Thread thread = runAppMaster(appMaster);
-
- List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1, 1000);
-
- interruptedThread(thread);
-
- assertEquals(0, contRequests.size());
- }
-
- /**
- * @throws Exception If failed.
- */
- @Test
- public void testClusterResource() throws Exception {
- rmMock.availableRes(new MockResource(1024, 2));
-
- appMaster.setRmClient(rmMock);
- appMaster.setNmClient(new NMMock());
-
- props.cpusPerNode(8);
- props.memoryPerNode(10240);
- props.instances(3);
-
- Thread thread = runAppMaster(appMaster);
-
- List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1, 1000);
-
- interruptedThread(thread);
-
- assertEquals(0, contRequests.size());
- }
-
- /**
- * @throws Exception If failed.
- */
- @Test
- public void testClusterAllocatedResource() throws Exception {
- rmMock.availableRes(new MockResource(1024, 2));
-
- appMaster.setRmClient(rmMock);
- appMaster.setNmClient(new NMMock());
-
- appMaster.setFs(new MockFileSystem());
-
- props.cpusPerNode(8);
- props.memoryPerNode(5000);
- props.instances(3);
-
- // Check that container resources
- appMaster.onContainersAllocated(Collections.singletonList(createContainer("simple", 5, 2000)));
- assertEquals(0, appMaster.getContainers().size());
-
- appMaster.onContainersAllocated(Collections.singletonList(createContainer("simple", 10, 2000)));
- assertEquals(0, appMaster.getContainers().size());
-
- appMaster.onContainersAllocated(Collections.singletonList(createContainer("simple", 1, 7000)));
- assertEquals(0, appMaster.getContainers().size());
-
- appMaster.onContainersAllocated(Collections.singletonList(createContainer("simple", 8, 5000)));
- assertEquals(1, appMaster.getContainers().size());
-
- appMaster.onContainersAllocated(Collections.singletonList(createContainer("simple", 10, 7000)));
- assertEquals(2, appMaster.getContainers().size());
- }
-
- /**
- * @throws Exception If failed.
- */
- @Test
- public void testStartReleaseContainer() throws Exception {
- rmMock.availableRes(new MockResource(1024, 2));
-
- NMMock nmClient = new NMMock();
-
- appMaster.setRmClient(rmMock);
- appMaster.setNmClient(nmClient);
-
- appMaster.setFs(new MockFileSystem());
-
- props.cpusPerNode(8);
- props.memoryPerNode(5000);
- props.instances(3);
-
- // Check that container resources
- appMaster.onContainersAllocated(Collections.singletonList(createContainer("simple", 5, 2000)));
- assertEquals(1, rmMock.releasedResources().size());
-
- appMaster.onContainersAllocated(Collections.singletonList(createContainer("simple", 5, 7000)));
- assertEquals(2, rmMock.releasedResources().size());
-
- appMaster.onContainersAllocated(Collections.singletonList(createContainer("simple", 9, 2000)));
- assertEquals(3, rmMock.releasedResources().size());
-
- appMaster.onContainersAllocated(Collections.singletonList(createContainer("simple", 8, 5000)));
- assertEquals(3, rmMock.releasedResources().size());
- assertEquals(1, nmClient.startedContainer().size());
- }
-
-
- /**
- * @throws Exception If failed.
- */
- @Test
- public void testHostnameConstraint() throws Exception {
- rmMock.availableRes(new MockResource(1024, 2));
-
- NMMock nmClient = new NMMock();
-
- appMaster.setRmClient(rmMock);
- appMaster.setNmClient(nmClient);
-
- appMaster.setFs(new MockFileSystem());
-
- props.cpusPerNode(8);
- props.memoryPerNode(5000);
- props.instances(3);
- props.hostnameConstraint(Pattern.compile("ignoreHost"));
-
- // Check that container resources
- appMaster.onContainersAllocated(Collections.singletonList(createContainer("simple", 8, 5000)));
- assertEquals(0, rmMock.releasedResources().size());
- assertEquals(1, nmClient.startedContainer().size());
-
- appMaster.onContainersAllocated(Collections.singletonList(createContainer("ignoreHost", 8, 5000)));
- assertEquals(1, rmMock.releasedResources().size());
- assertEquals(1, nmClient.startedContainer().size());
- }
-
- /**
- * @throws Exception If failed.
- */
- @Test
- public void testContainerEnvironment() throws Exception {
- props.memoryPerNode(1001);
- props.memoryOverHeadPerNode(2002);
-
- // Properties are used to initialize AM container environment
- Map<String, String> result = props.toEnvs();
- assertEquals(1001, (int)Double.parseDouble(result.get(ClusterProperties.IGNITE_MEMORY_PER_NODE)));
- assertEquals(2002, (int)Double.parseDouble(result.get(ClusterProperties.IGNITE_MEMORY_OVERHEAD_PER_NODE)));
- }
-
- /**
- * @throws Exception If failed.
- */
- @Test
- public void testQueue() throws Exception {
- // Default Queue check
- Map<String, String> result = props.toEnvs();
- assertEquals(ClusterProperties.DEFAULT_IGNITE_YARN_QUEUE, result.get(ClusterProperties.IGNITE_YARN_QUEUE));
-
- props.yarnQueue("ignite");
- result = props.toEnvs();
- assertEquals("ignite", result.get(ClusterProperties.IGNITE_YARN_QUEUE));
- }
-
- /**
- * @param host Host.
- * @param cpu Cpu count.
- * @param mem Memory.
- * @return Container.
- */
- private Container createContainer(String host, int cpu, int mem) {
- return Container.newInstance(
- ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0),
- ThreadLocalRandom.current().nextLong()),
- NodeId.newInstance(host, 0),
- "example.com",
- new MockResource(mem, cpu),
- Priority.newInstance(0),
- null
- );
- }
-
- /**
- * @param rmMock RM mock.
- * @param expectedCnt Expected cnt.
- * @param timeOut Timeout.
- * @return Requests.
- */
- private List<AMRMClient.ContainerRequest> collectRequests(RMMock rmMock, int expectedCnt, int timeOut) {
- long startTime = System.currentTimeMillis();
-
- List<AMRMClient.ContainerRequest> requests = rmMock.requests();
-
- while (requests.size() < expectedCnt
- && (System.currentTimeMillis() - startTime) < timeOut)
- requests = rmMock.requests();
-
- return requests;
- }
-
- /**
- * Runs appMaster other thread.
- *
- * @param appMaster Application master.
- * @return Thread.
- */
- private static Thread runAppMaster(final ApplicationMaster appMaster) {
- Thread thread = new Thread(new Runnable() {
- @Override public void run() {
- try {
- appMaster.run();
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- });
-
- thread.start();
-
- return thread;
- }
-
- /**
- * Interrupt thread and join.
- *
- * @param thread Thread.
- */
- private static void interruptedThread(Thread thread) throws InterruptedException {
- thread.interrupt();
-
- thread.join();
- }
-
- /**
- * Resource manager mock.
- */
- private static class RMMock extends AMRMClientAsync {
- /** */
- private List<AMRMClient.ContainerRequest> contRequests = new ArrayList<>();
-
- /** */
- private List<ContainerId> releasedConts = new ArrayList<>();
-
- /** */
- private Resource availableRes;
-
- /** */
- public RMMock() {
- super(0, null);
- }
-
- /**
- * @return Requests.
- */
- public List<AMRMClient.ContainerRequest> requests() {
- return contRequests;
- }
-
- /**
- * @return Released resources.
- */
- public List<ContainerId> releasedResources() {
- return releasedConts;
- }
-
- /**
- * Sets resource.
- *
- * @param availableRes Available resource.
- */
- public void availableRes(Resource availableRes) {
- this.availableRes = availableRes;
- }
-
- /**
- * Clear internal state.
- */
- public void clear() {
- contRequests.clear();
- releasedConts.clear();
- availableRes = null;
- }
-
- /** {@inheritDoc} */
- @Override public List<? extends Collection> getMatchingRequests(Priority priority, String resourceName,
- Resource capability) {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public RegisterApplicationMasterResponse registerApplicationMaster(String appHostName,
- int appHostPort, String appTrackingUrl) throws YarnException, IOException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void unregisterApplicationMaster(FinalApplicationStatus appStatus, String appMessage,
- String appTrackingUrl) throws YarnException, IOException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void addContainerRequest(AMRMClient.ContainerRequest req) {
- contRequests.add(req);
- }
-
- /** {@inheritDoc} */
- @Override public void removeContainerRequest(AMRMClient.ContainerRequest req) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void requestContainerUpdate(Container container, UpdateContainerRequest request) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void releaseAssignedContainer(ContainerId containerId) {
- releasedConts.add(containerId);
- }
-
- /** {@inheritDoc} */
- @Override public Resource getAvailableResources() {
- return availableRes;
- }
-
- /** {@inheritDoc} */
- @Override public int getClusterNodeCount() {
- return 0;
- }
-
- /**
- * Update application's blacklist with addition or removal resources.
- *
- * @param blacklistAdditions list of resources which should be added to the
- * application blacklist
- * @param blacklistRemovals list of resources which should be removed from the
- * application blacklist
- */
- @Override public void updateBlacklist(List blacklistAdditions, List blacklistRemovals) {
- // No-op.
- }
- }
-
- /**
- * Network manager mock.
- */
- public static class NMMock extends NMClient {
- /** */
- private List<ContainerLaunchContext> startedContainer = new ArrayList<>();
-
- /** */
- public NMMock() {
- super("name");
- }
-
- /**
- * @return Started containers.
- */
- public List<ContainerLaunchContext> startedContainer() {
- return startedContainer;
- }
-
- /** {@inheritDoc} */
- @Override public Map<String, ByteBuffer> startContainer(Container container,
- ContainerLaunchContext containerLaunchContext) throws YarnException, IOException {
-
- startedContainer.add(containerLaunchContext);
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void increaseContainerResource(Container container) throws YarnException, IOException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void updateContainerResource(Container container) throws YarnException, IOException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void stopContainer(ContainerId containerId, NodeId nodeId) throws YarnException, IOException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public ContainerStatus getContainerStatus(ContainerId containerId, NodeId nodeId)
- throws YarnException, IOException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void reInitializeContainer(ContainerId id, ContainerLaunchContext context,
- boolean b) throws YarnException, IOException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void restartContainer(ContainerId id) throws YarnException, IOException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void rollbackLastReInitialization(ContainerId id) throws YarnException, IOException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void commitLastReInitialization(ContainerId id) throws YarnException, IOException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void cleanupRunningContainersOnStop(boolean enabled) {
- // No-op.
- }
- }
-
- /**
- * Resource.
- */
- public static class MockResource extends Resource {
- /** Memory. */
- private int mem;
-
- /** CPU. */
- private int cpu;
-
- /**
- * @param mem Memory.
- * @param cpu CPU.
- */
- public MockResource(int mem, int cpu) {
- this.mem = mem;
- this.cpu = cpu;
- }
-
- /** {@inheritDoc} */
- @Override public int getMemory() {
- return mem;
- }
-
- /** {@inheritDoc} */
- @Override public void setMemory(int memory) {
- this.mem = memory;
- }
-
- /** {@inheritDoc} */
- @Override public int getVirtualCores() {
- return cpu;
- }
-
- /** {@inheritDoc} */
- @Override public void setVirtualCores(int vCores) {
- this.cpu = vCores;
- }
-
- /** {@inheritDoc} */
- @Override public int compareTo(Resource resource) {
- return 0;
- }
- }
-
- /**
- * Mock file system.
- */
- public static class MockFileSystem extends FileSystem {
- /** */
- public MockFileSystem() {
- }
-
- /** {@inheritDoc} */
- @Override public Path makeQualified(Path path) {
- return new Path("/test/path");
- }
-
- /** {@inheritDoc} */
- @Override public FileStatus getFileStatus(Path f) throws IOException {
- return new FileStatus();
- }
-
- /** {@inheritDoc} */
- @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public Path getWorkingDirectory() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void setWorkingDirectory(Path new_dir) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException {
- return new FileStatus[0];
- }
-
- /** {@inheritDoc} */
- @Override public boolean delete(Path f, boolean recursive) throws IOException {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public boolean rename(Path src, Path dst) throws IOException {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize,
- short replication, long blockSize, Progressable progress) throws IOException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public URI getUri() {
- return null;
- }
- }
-}
diff --git a/parent-internal/pom.xml b/parent-internal/pom.xml
index b31009d223b..6315418976d 100644
--- a/parent-internal/pom.xml
+++ b/parent-internal/pom.xml
@@ -218,10 +218,6 @@
<title>Mesos Framework</title>
<packages>org.apache.ignite.mesos*</packages>
</group>
- <group>
- <title>Yarn Integration</title>
- <packages>org.apache.ignite.yarn*</packages>
- </group>
<group>
<title>Kubernetes Integration</title>
<packages>org.apache.ignite.kubernetes*</packages>
diff --git a/pom.xml b/pom.xml
index 558fdc6ffea..a2f330f0a5a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -61,7 +61,6 @@
<module>modules/jcl</module>
<module>modules/codegen</module>
<module>modules/cloud</module>
- <module>modules/yarn</module>
<module>modules/zookeeper</module>
<module>modules/osgi-paxlogging</module>
<module>modules/osgi-karaf</module>