You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/07/21 18:39:44 UTC
[42/52] [abbrv] hadoop git commit: YARN-6613. Update json validation
for new native services providers. Contributed by Billie Rinaldi
YARN-6613. Update json validation for new native services providers. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb984564
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb984564
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb984564
Branch: refs/heads/yarn-native-services
Commit: eb9845649a372e514088b3e9dfcc4ed575c43110
Parents: 081a227
Author: Jian He <ji...@apache.org>
Authored: Thu May 25 12:47:19 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Fri Jul 21 11:38:14 2017 -0700
----------------------------------------------------------------------
.../hadoop-yarn-services-api/pom.xml | 57 +--
...RN-Simplified-V1-API-Layer-For-Services.yaml | 12 +-
.../api/impl/TestApplicationApiService.java | 209 ----------
.../apache/slider/api/resource/Application.java | 4 +
.../apache/slider/api/resource/Component.java | 41 +-
.../slider/api/resource/Configuration.java | 10 +-
.../org/apache/slider/client/SliderClient.java | 109 ++---
.../slider/client/SliderYarnClientImpl.java | 61 +++
.../org/apache/slider/common/SliderKeys.java | 128 +-----
.../slider/common/tools/CoreFileSystem.java | 64 ---
.../apache/slider/common/tools/SliderUtils.java | 14 +-
.../slider/core/persist/InstancePaths.java | 58 ---
.../providers/AbstractClientProvider.java | 51 ++-
.../slider/providers/SliderProviderFactory.java | 12 +-
.../tarball/TarballProviderService.java | 2 +-
.../server/appmaster/SliderAppMaster.java | 18 +-
.../slider/util/RestApiErrorMessages.java | 4 +-
.../org/apache/slider/util/ServiceApiUtil.java | 273 +++++++------
.../slider/client/TestKeytabCommandOptions.java | 11 +-
.../common/tools/TestMiscSliderUtils.java | 49 ---
.../apache/slider/core/conf/ExampleAppJson.java | 64 +++
.../slider/core/conf/ExampleConfResources.java | 58 ---
.../core/conf/TestConfTreeLoadExamples.java | 64 ---
.../core/conf/TestConfigurationResolve.java | 146 ++++++-
.../slider/core/conf/TestExampleAppJson.java | 79 ++++
.../providers/TestAbstractClientProvider.java | 121 ++++++
.../TestBuildApplicationComponent.java | 96 +++++
.../slider/providers/TestDefaultProvider.java | 60 +++
.../model/appstate/BaseMockAppStateAATest.java | 2 +-
.../appstate/TestMockAppStateAAPlacement.java | 2 +-
.../TestMockAppStateContainerFailure.java | 2 +-
.../TestMockAppStateFlexDynamicRoles.java | 5 +-
.../TestMockAppStateRebuildOnAMRestart.java | 2 +-
.../appstate/TestMockAppStateUniqueNames.java | 3 +-
.../TestMockContainerResourceAllocations.java | 2 +-
.../model/mock/BaseMockAppStateTest.java | 14 +-
.../appmaster/model/mock/MockFactory.java | 3 +
.../apache/slider/utils/TestServiceApiUtil.java | 393 +++++++++++++++++++
.../slider/utils/YarnMiniClusterTestBase.java | 99 +++--
.../slider/utils/YarnZKMiniClusterTestBase.java | 5 +-
.../conf/examples/app-override-resolved.json | 49 ---
.../slider/core/conf/examples/app-override.json | 33 +-
.../slider/core/conf/examples/app-resolved.json | 81 ----
.../apache/slider/core/conf/examples/app.json | 13 +-
.../slider/core/conf/examples/default.json | 16 +
.../slider/core/conf/examples/external0.json | 8 +
.../slider/core/conf/examples/external1.json | 30 ++
.../slider/core/conf/examples/external2.json | 22 ++
48 files changed, 1539 insertions(+), 1120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index 4e88aef..bc714db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -28,11 +28,6 @@
<packaging>jar</packaging>
<description>Hadoop YARN REST APIs for services</description>
- <properties>
- <test.failIfNoTests>false</test.failIfNoTests>
- <powermock.version>1.6.5</powermock.version>
- </properties>
-
<build>
<!-- resources are filtered for dynamic updates. This gets build info in-->
@@ -81,30 +76,10 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
- <version>${maven-surefire-plugin.version}</version>
<configuration>
- <reuseForks>${test.reuseForks}</reuseForks>
- <forkMode>${test.forkMode}</forkMode>
- <forkCount>1</forkCount>
- <forkedProcessTimeoutInSeconds>${test.forkedProcessTimeoutInSeconds}
- </forkedProcessTimeoutInSeconds>
- <threadCount>1</threadCount>
- <argLine>${test.argLine}</argLine>
- <failIfNoTests>${test.failIfNoTests}</failIfNoTests>
- <redirectTestOutputToFile>${build.redirect.test.output.to.file}</redirectTestOutputToFile>
<environmentVariables>
- <PATH>${test.env.path}</PATH>
+ <JAVA_HOME>${java.home}</JAVA_HOME>
</environmentVariables>
- <systemPropertyVariables>
- <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
- <java.awt.headless>true</java.awt.headless>
- </systemPropertyVariables>
- <includes>
- <include>**/Test*.java</include>
- </includes>
- <excludes>
- <exclude>**/Test*$*.java</exclude>
- </excludes>
</configuration>
</plugin>
@@ -121,13 +96,6 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- <version>${project.version}</version>
- </dependency>
- <dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-annotations</artifactId>
</dependency>
@@ -147,29 +115,6 @@
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
</dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>${powermock.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-easymock</artifactId>
- <version>${powermock.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 82cc30f..f8ed4d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -158,7 +158,7 @@ definitions:
type: string
description: A unique application id.
artifact:
- description: Artifact of single-component applications. Mandatory if components attribute is not specified.
+ description: Artifact of single-component applications.
$ref: '#/definitions/Artifact'
resource:
description: Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level.
@@ -230,16 +230,16 @@ definitions:
type: string
description: Assigns an app to a named partition of the cluster where the application desires to run (optional). If not specified all apps are submitted to a default label of the app owner. One or more labels can be setup for each application owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.
Artifact:
- description: Artifact of an application component.
+ description: Artifact of an application component. If not specified, component will just run the bare launch command and no artifact will be localized.
required:
- id
properties:
id:
type: string
- description: Artifact id. Examples are package location uri for tarball based apps, image name for docker, etc.
+ description: Artifact id. Examples are package location uri for tarball based apps, image name for docker, name of application, etc.
type:
type: string
- description: Artifact type, like docker, tarball, etc. (optional).
+ description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For APPLICATION type, the application specified will be read and its components will be added into this application. The original component with artifact type APPLICATION will be removed (any properties specified in the original component will be ignored).
enum:
- DOCKER
- TARBALL
@@ -269,7 +269,7 @@ definitions:
$ref: '#/definitions/Artifact'
launch_command:
type: string
- description: The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).
+ description: The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any).
resource:
description: Resource of this component (optional). If not specified, the application level global resource takes effect.
$ref: '#/definitions/Resource'
@@ -344,7 +344,7 @@ definitions:
- HADOOP_XML
dest_file:
type: string
- description: The absolute path that this configuration file should be mounted as, in the application container.
+ description: The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf.
src_file:
type: string
description: This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
deleted file mode 100644
index 6e077d2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.services.api.impl;
-
-import static org.apache.slider.util.RestApiConstants.*;
-import static org.apache.slider.util.RestApiErrorMessages.*;
-
-import java.util.ArrayList;
-
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.Resource;
-import org.apache.slider.util.ServiceApiUtil;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
-import org.powermock.modules.junit4.PowerMockRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test class for application life time monitor feature test.
- */
-@RunWith(PowerMockRunner.class)
-@SuppressStaticInitializationFor("org.apache.hadoop.yarn.services.api.impl.ApplicationApiService")
-public class TestApplicationApiService {
- private static final Logger logger = LoggerFactory
- .getLogger(TestApplicationApiService.class);
- private static String EXCEPTION_PREFIX = "Should have thrown exception: ";
- private static String NO_EXCEPTION_PREFIX = "Should not have thrown exception: ";
- private ApplicationApiService appApiService;
-
- @Before
- public void setup() throws Exception {
- appApiService = new ApplicationApiService();
- }
-
- @After
- public void tearDown() throws Exception {
- }
-
- @Test(timeout = 90000)
- public void testValidateApplicationPostPayload() throws Exception {
- Application app = new Application();
-
- // no name
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX + "application with no name");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
- }
-
- // bad format name
- String[] badNames = { "4finance", "Finance", "finance@home" };
- for (String badName : badNames) {
- app.setName(badName);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID_FORMAT,
- e.getMessage());
- }
- }
-
- // no artifact
- app.setName("finance_home");
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_ARTIFACT_INVALID, e.getMessage());
- }
-
- // no artifact id
- Artifact artifact = new Artifact();
- app.setArtifact(artifact);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
- }
-
- // if artifact is of type APPLICATION then everything is valid here
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- artifact.setId("app.io/hbase:facebook_0.2");
- app.setNumberOfContainers(5l);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- } catch (IllegalArgumentException e) {
- logger.error("application attributes specified should be valid here", e);
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- // default-component, default-lifetime and the property component_type
- // should get assigned here
- Assert.assertEquals(app.getComponents().get(0).getName(),
- DEFAULT_COMPONENT_NAME);
- Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
- //TODO handle external app
-
- // unset artifact type, default component and no of containers to test other
- // validation logic
- artifact.setType(null);
- app.setComponents(new ArrayList<>());
- app.setNumberOfContainers(null);
-
- // resource not specified
- artifact.setId("docker.io/centos:centos7");
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX + "application with no resource");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_RESOURCE_INVALID, e.getMessage());
- }
-
- // memory not specified
- Resource res = new Resource();
- app.setResource(res);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX + "application with no memory");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_RESOURCE_MEMORY_INVALID, e.getMessage());
- }
-
- // cpu does not need to be always specified, it's an optional feature in yarn
- // invalid no of cpus
- res.setMemory("100mb");
- res.setCpus(-2);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(
- EXCEPTION_PREFIX + "application with invalid no of cpups");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID_RANGE, e.getMessage());
- }
-
- // number of containers not specified
- res.setCpus(2);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(
- EXCEPTION_PREFIX + "application with no container count");
- } catch (IllegalArgumentException e) {
- Assert.assertTrue(e.getMessage().contains(ERROR_CONTAINERS_COUNT_INVALID));
- }
-
- // specifying profile along with cpus/memory raises exception
- res.setProfile("hbase_finance_large");
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX
- + "application with resource profile along with cpus/memory");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED,
- e.getMessage());
- }
-
- // currently resource profile alone is not supported.
- // TODO: remove the next test once it is supported.
- res.setCpus(null);
- res.setMemory(null);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- Assert.fail(EXCEPTION_PREFIX
- + "application with resource profile only - NOT SUPPORTED");
- } catch (IllegalArgumentException e) {
- Assert.assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
- e.getMessage());
- }
-
- // unset profile here and add cpus/memory back
- res.setProfile(null);
- res.setCpus(2);
- res.setMemory("2gb");
-
- // everything valid here
- app.setNumberOfContainers(5l);
- try {
- ServiceApiUtil.validateApplicationPayload(app, null);
- } catch (IllegalArgumentException e) {
- logger.error("application attributes specified should be valid here", e);
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- // Now test with components
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java
index 502b519..4b7e59b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java
@@ -286,6 +286,10 @@ public class Application extends BaseResource {
this.components = components;
}
+ public void addComponent(Component component) {
+ components.add(component);
+ }
+
public Component getComponent(String name) {
for (Component component : components) {
if (component.getName().equals(name)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java
index e7f3796..229e288 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java
@@ -50,9 +50,9 @@ public class Component implements Serializable {
private String name = null;
private List<String> dependencies = new ArrayList<String>();
private ReadinessCheck readinessCheck = null;
- private Artifact artifact = new Artifact();
+ private Artifact artifact = null;
private String launchCommand = null;
- private Resource resource = new Resource();
+ private Resource resource = null;
private Long numberOfContainers = null;
private Boolean uniqueComponentSupport = false;
private Boolean runPrivilegedContainer = false;
@@ -406,4 +406,41 @@ public class Component implements Serializable {
}
return o.toString().replace("\n", "\n ");
}
+
+ /**
+ * Merge from another component into this component without overwriting.
+ */
+ public void mergeFrom(Component that) {
+ if (this.getArtifact() == null) {
+ this.setArtifact(that.getArtifact());
+ }
+ if (this.getResource() == null) {
+ this.setResource(that.getResource());
+ }
+ if (this.getNumberOfContainers() == null) {
+ this.setNumberOfContainers(that.getNumberOfContainers());
+ }
+ if (this.getLaunchCommand() == null) {
+ this.setLaunchCommand(that.getLaunchCommand());
+ }
+ this.getConfiguration().mergeFrom(that.getConfiguration());
+ if (this.getQuicklinks() == null) {
+ this.setQuicklinks(that.getQuicklinks());
+ }
+ if (this.getRunPrivilegedContainer() == null) {
+ this.setRunPrivilegedContainer(that.getRunPrivilegedContainer());
+ }
+ if (this.getUniqueComponentSupport() == null) {
+ this.setUniqueComponentSupport(that.getUniqueComponentSupport());
+ }
+ if (this.getDependencies() == null) {
+ this.setDependencies(that.getDependencies());
+ }
+ if (this.getPlacementPolicy() == null) {
+ this.setPlacementPolicy(that.getPlacementPolicy());
+ }
+ if (this.getReadinessCheck() == null) {
+ this.setReadinessCheck(that.getReadinessCheck());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
index 37d1a40..e89306c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import org.apache.commons.lang.StringUtils;
+import org.apache.slider.common.tools.SliderUtils;
import java.io.Serializable;
import java.util.ArrayList;
@@ -197,8 +198,10 @@ public class Configuration implements Serializable {
* this ConfigFile.
*/
public synchronized void mergeFrom(Configuration that) {
- this.properties.putAll(that.getProperties());
- this.env.putAll(that.getEnv());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
+ .getProperties());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
+
Map<String, ConfigFile> thatMap = new HashMap<>();
for (ConfigFile file : that.getFiles()) {
thatMap.put(file.getDestFile(), file.copy());
@@ -206,7 +209,8 @@ public class Configuration implements Serializable {
for (ConfigFile thisFile : files) {
if(thatMap.containsKey(thisFile.getDestFile())) {
ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
- thisFile.getProps().putAll(thatFile.getProps());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProps(),
+ thatFile.getProps());
thatMap.remove(thisFile.getDestFile());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 32d78b4..29ca471 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -19,7 +19,6 @@
package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.io.Files;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -101,6 +100,7 @@ import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.params.ClientArgs;
import org.apache.slider.common.params.CommonArgs;
import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.common.tools.SliderVersionInfo;
@@ -142,8 +142,6 @@ import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.codehaus.jackson.map.PropertyNamingStrategy;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -636,16 +634,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionBuild(Application application) throws YarnException,
IOException {
Path appDir = checkAppNotExistOnHdfs(application);
+ ServiceApiUtil.validateAndResolveApplication(application, sliderFileSystem);
persistApp(appDir, application);
+ deployedClusterName = application.getName();
return EXIT_SUCCESS;
}
public ApplicationId actionCreate(Application application)
throws IOException, YarnException {
- ServiceApiUtil.validateApplicationPayload(application,
- sliderFileSystem.getFileSystem());
String appName = application.getName();
validateClusterName(appName);
+ ServiceApiUtil.validateAndResolveApplication(application, sliderFileSystem);
verifyNoLiveApp(appName, "Create");
Path appDir = checkAppNotExistOnHdfs(application);
@@ -880,6 +879,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return appDir;
}
+ private Path checkAppExistOnHdfs(String appName)
+ throws IOException, SliderException {
+ Path appDir = sliderFileSystem.buildClusterDirPath(appName);
+ sliderFileSystem.verifyPathExists(
+ new Path(appDir, appName + ".json"));
+ return appDir;
+ }
+
private void persistApp(Path appDir, Application application)
throws IOException, SliderException {
FsPermission appDirPermission = new FsPermission("750");
@@ -1125,7 +1132,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
YarnException,
IOException {
if (clientInfo.install) {
- return doClientInstall(clientInfo);
+ // TODO implement client install
+ throw new UnsupportedOperationException("Client install not yet " +
+ "supported");
} else {
throw new BadCommandArgumentsException(
"Only install, keystore, and truststore commands are supported for the client.\n"
@@ -1134,66 +1143,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
- private int doClientInstall(ActionClientArgs clientInfo)
- throws IOException, SliderException {
-
- require(clientInfo.installLocation != null,
- E_INVALID_INSTALL_LOCATION +"\n"
- + CommonArgs.usage(serviceArgs, ACTION_CLIENT));
- require(clientInfo.installLocation.exists(),
- E_INSTALL_PATH_DOES_NOT_EXIST + ": " + clientInfo.installLocation.getAbsolutePath());
-
- require(clientInfo.installLocation.isDirectory(),
- E_INVALID_INSTALL_PATH + ": " + clientInfo.installLocation.getAbsolutePath());
-
- File pkgFile;
- File tmpDir = null;
-
- require(isSet(clientInfo.packageURI) || isSet(clientInfo.name),
- E_INVALID_APPLICATION_PACKAGE_LOCATION);
- if (isSet(clientInfo.packageURI)) {
- pkgFile = new File(clientInfo.packageURI);
- } else {
- Path appDirPath = sliderFileSystem.buildAppDefDirPath(clientInfo.name);
- Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
- require(sliderFileSystem.isFile(appDefPath),
- E_INVALID_APPLICATION_PACKAGE_LOCATION);
- tmpDir = Files.createTempDir();
- pkgFile = new File(tmpDir, SliderKeys.DEFAULT_APP_PKG);
- sliderFileSystem.copyHdfsFileToLocal(appDefPath, pkgFile);
- }
- require(pkgFile.isFile(),
- E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + " at %s", pkgFile.getAbsolutePath());
-
- JSONObject config = null;
- if(clientInfo.clientConfig != null) {
- try {
- byte[] encoded = Files.toByteArray(clientInfo.clientConfig);
- config = new JSONObject(new String(encoded, Charset.defaultCharset()));
- } catch (JSONException jsonEx) {
- log.error("Unable to read supplied configuration at {}: {}",
- clientInfo.clientConfig, jsonEx);
- log.debug("Unable to read supplied configuration at {}: {}",
- clientInfo.clientConfig, jsonEx, jsonEx);
- throw new BadConfigException(E_MUST_BE_A_VALID_JSON_FILE, jsonEx);
- }
- }
-
- // TODO handle client install
- // Only INSTALL is supported
- // ClientProvider
- // provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
- // provider.processClientOperation(sliderFileSystem,
- // getRegistryOperations(),
- // getConfig(),
- // "INSTALL",
- // clientInfo.installLocation,
- // pkgFile,
- // config,
- // clientInfo.name);
- return EXIT_SUCCESS;
- }
-
@Override
public int actionUpdate(String clustername,
AbstractClusterBuildingActionArgs buildInfo) throws
@@ -1802,23 +1751,24 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionStart(String appName, ActionThawArgs thaw)
throws YarnException, IOException {
validateClusterName(appName);
+ Path appDir = checkAppExistOnHdfs(appName);
+ Application application = ServiceApiUtil.loadApplication(sliderFileSystem,
+ appName);
+ ServiceApiUtil.validateAndResolveApplication(application, sliderFileSystem);
// see if it is actually running and bail out;
verifyNoLiveApp(appName, "Thaw");
- Path appDir = sliderFileSystem.buildClusterDirPath(appName);
- Path appJson = new Path(appDir, appName + ".json");
- Application application =
- jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson);
- submitApp(application);
+ ApplicationId appId = submitApp(application);
+ application.setId(appId.toString());
+ // write app definition on to hdfs
+ persistApp(appDir, application);
return 0;
}
public Map<String, Long> flex(String appName, Map<String, Long>
componentCounts) throws YarnException, IOException {
validateClusterName(appName);
- Path appDir = sliderFileSystem.buildClusterDirPath(appName);
- Path appJson = new Path(appDir, appName + ".json");
- Application persistedApp =
- jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson);
+ Application persistedApp = ServiceApiUtil.loadApplication(sliderFileSystem,
+ appName);
Map<String, Long> original = new HashMap<>(componentCounts.size());
for (Component persistedComp : persistedApp.getComponents()) {
String name = persistedComp.getName();
@@ -1833,7 +1783,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
+ " do not exist in app definition.");
}
jsonSerDeser
- .save(sliderFileSystem.getFileSystem(), appJson, persistedApp, true);
+ .save(sliderFileSystem.getFileSystem(), ServiceApiUtil.getAppJsonPath(
+ sliderFileSystem, appName), persistedApp, true);
log.info("Updated app definition file for components " + componentCounts
.keySet());
@@ -2705,6 +2656,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
yarnClient);
}
+ @VisibleForTesting
+ public ApplicationReport monitorAppToRunning(Duration duration)
+ throws YarnException, IOException {
+ return yarnClient.monitorAppToState(applicationId, YarnApplicationState
+ .RUNNING, duration);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 4839395..306bd99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -245,4 +245,65 @@ public class SliderYarnClientImpl extends YarnClientImpl {
}
return results;
}
+
+ /**
+ * Monitor the submitted application for reaching the requested state.
+ * Will also report if the app reaches a later state (failed, killed, etc)
+ * Kill application if duration!= null & time expires.
+ * @param appId Application Id of application to be monitored
+ * @param duration how long to wait -must be more than 0
+ * @param desiredState desired state.
+ * @return the application report -null on a timeout
+ * @throws YarnException
+ * @throws IOException
+ */
+ public ApplicationReport monitorAppToState(
+ ApplicationId appId, YarnApplicationState desiredState, Duration duration)
+ throws YarnException, IOException {
+
+ if (appId == null) {
+ throw new BadCommandArgumentsException("null application ID");
+ }
+ if (duration.limit <= 0) {
+ throw new BadCommandArgumentsException("Invalid monitoring duration");
+ }
+ log.debug("Waiting {} millis for app to reach state {} ",
+ duration.limit,
+ desiredState);
+ duration.start();
+ try {
+ while (true) {
+ // Get application report for the appId we are interested in
+
+ ApplicationReport r = getApplicationReport(appId);
+
+ log.debug("queried status is\n{}",
+ new SliderUtils.OnDemandReportStringifier(r));
+
+ YarnApplicationState state = r.getYarnApplicationState();
+ if (state.ordinal() >= desiredState.ordinal()) {
+ log.debug("App in desired state (or higher) :{}", state);
+ return r;
+ }
+ if (duration.getLimitExceeded()) {
+ log.debug(
+ "Wait limit of {} millis to get to state {}, exceeded; app " +
+ "status\n {}",
+ duration.limit,
+ desiredState,
+ new SliderUtils.OnDemandReportStringifier(r));
+ return null;
+ }
+
+ // sleep 1s.
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ignored) {
+ log.debug("Thread sleep in monitoring loop interrupted");
+ }
+ }
+ } finally {
+ duration.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 734fec5..865562e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -18,10 +18,6 @@
package org.apache.slider.common;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
/**
* Keys and various constants for Slider
*/
@@ -73,17 +69,10 @@ public interface SliderKeys extends SliderXmlConfKeys {
String APP_TYPE = "org-apache-slider";
/**
- * Key for component type. This MUST NOT be set in app_config/global {@value}
- */
- String COMPONENT_TYPE_KEY = "site.global.component_type";
- /**
* A component type for an external app that has been predefined using the
* slider build command
*/
- String COMPONENT_TYPE_EXTERNAL_APP = "external_app";
String COMPONENT_SEPARATOR = "-";
- List<String> COMPONENT_KEYS_TO_SKIP = Collections.unmodifiableList(Arrays
- .asList("zookeeper.", "env.MALLOC_ARENA_MAX", "site.fs.", "site.dfs."));
/**
* A component type for a client component
@@ -91,48 +80,19 @@ public interface SliderKeys extends SliderXmlConfKeys {
String COMPONENT_TYPE_CLIENT = "client";
/**
- * Key for application version. This must be set in app_config/global {@value}
+ * Key for application version.
*/
- String APP_VERSION = "site.global.app_version";
String APP_VERSION_UNKNOWN = "awaiting heartbeat...";
/**
* Keys for application container specific properties, like release timeout
*/
String APP_CONTAINER_RELEASE_TIMEOUT = "site.global.app_container.release_timeout_secs";
- int APP_CONTAINER_HEARTBEAT_INTERVAL_SEC = 10; // look for HEARTBEAT_IDDLE_INTERVAL_SEC
/**
- * JVM arg to force IPv4 {@value}
- */
- String JVM_ENABLE_ASSERTIONS = "-ea";
-
- /**
- * JVM arg enable JVM system/runtime {@value}
+ * Subdirectories of HDFS cluster dir.
*/
- String JVM_ENABLE_SYSTEM_ASSERTIONS = "-esa";
-
- /**
- * JVM arg to force IPv4 {@value}
- */
- String JVM_FORCE_IPV4 = "-Djava.net.preferIPv4Stack=true";
-
- /**
- * JVM arg to go headless {@value}
- */
-
- String JVM_JAVA_HEADLESS = "-Djava.awt.headless=true";
-
- /**
- * This is the name of the dir/subdir containing
- * the hbase conf that is propagated via YARN
- * {@value}
- */
- String PROPAGATED_CONF_DIR_NAME = "propagatedconf";
- String INFRA_DIR_NAME = "infra";
- String GENERATED_CONF_DIR_NAME = "generated";
- String SNAPSHOT_CONF_DIR_NAME = "snapshot";
- String DATA_DIR_NAME = "database";
+ String DATA_DIR_NAME = "data";
String HISTORY_DIR_NAME = "history";
String HISTORY_FILENAME_SUFFIX = "json";
String HISTORY_FILENAME_PREFIX = "rolehistory-";
@@ -159,14 +119,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
String CLUSTER_DIRECTORY = "cluster";
- String PACKAGE_DIRECTORY = "package";
-
- /**
- * JVM property to define the slider configuration directory;
- * this is set by the slider script: {@value}
- */
- String PROPERTY_CONF_DIR = "slider.confdir";
-
/**
* JVM property to define the slider lib directory;
* this is set by the slider script: {@value}
@@ -184,11 +136,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
String LOG4J_SERVER_PROP_FILENAME = "slideram-log4j.properties";
/**
- * Standard log4j file name : {@value}
- */
- String LOG4J_PROP_FILENAME = "log4j.properties";
-
- /**
* Log4j sysprop to name the resource :{@value}
*/
String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
@@ -209,9 +156,7 @@ public interface SliderKeys extends SliderXmlConfKeys {
*/
String SLIDER_SERVER_XML = "slider-server.xml";
- String TMP_LOGDIR_PREFIX = "/tmp/slider-";
String TMP_DIR_PREFIX = "tmp";
- String AM_DIR_PREFIX = "appmaster";
/**
* Store the default app definition, e.g. metainfo file or content of a folder
@@ -223,53 +168,11 @@ public interface SliderKeys extends SliderXmlConfKeys {
String ADDONS_DIR = "addons";
String SLIDER_JAR = "slider-core.jar";
- String JCOMMANDER_JAR = "jcommander.jar";
- String GSON_JAR = "gson.jar";
- String DEFAULT_APP_PKG = "appPkg.zip";
- String DEFAULT_JVM_HEAP = "256M";
- int DEFAULT_YARN_MEMORY = 256;
String STDOUT_AM = "slider-out.txt";
String STDERR_AM = "slider-err.txt";
- String DEFAULT_GC_OPTS = "";
String HADOOP_USER_NAME = "HADOOP_USER_NAME";
- String HADOOP_PROXY_USER = "HADOOP_PROXY_USER";
- String SLIDER_PASSPHRASE = "SLIDER_PASSPHRASE";
-
- boolean PROPAGATE_RESOURCE_OPTION = true;
-
- /**
- * Security associated keys.
- */
- String SECURITY_DIR = "security";
- String CRT_FILE_NAME = "ca.crt";
- String CSR_FILE_NAME = "ca.csr";
- String KEY_FILE_NAME = "ca.key";
- String KEYSTORE_FILE_NAME = "keystore.p12";
- String CRT_PASS_FILE_NAME = "pass.txt";
- String PASS_LEN = "50";
-
- String COMP_STORES_REQUIRED_KEY =
- "slider.component.security.stores.required";
- String COMP_KEYSTORE_PASSWORD_PROPERTY_KEY =
- "slider.component.keystore.password.property";
- String COMP_KEYSTORE_PASSWORD_ALIAS_KEY =
- "slider.component.keystore.credential.alias.property";
- String COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT =
- "component.keystore.credential.alias";
- String COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY =
- "slider.component.truststore.password.property";
- String COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY =
- "slider.component.truststore.credential.alias.property";
- String COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT =
- "component.truststore.credential.alias";
-
- /**
- * Python specific
- */
- String PYTHONPATH = "PYTHONPATH";
-
/**
* Name of the AM filter to use: {@value}
@@ -277,34 +180,11 @@ public interface SliderKeys extends SliderXmlConfKeys {
String AM_FILTER_NAME =
"org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer";
- /**
- * Allowed port range. This MUST be set in app_conf/global.
- * {@value}
- */
- String KEY_ALLOWED_PORT_RANGE = "site.global.slider.allowed.ports";
-
- /**
- * env var for custom JVM options.
- */
- String SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS";
-
- String SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA";
String YARN_CONTAINER_PATH = "/node/container/";
- String GLOBAL_CONFIG_TAG = "global";
- String SYSTEM_CONFIGS = "system_configs";
- String JAVA_HOME = "java_home";
- String TWO_WAY_SSL_ENABLED = "ssl.server.client.auth";
- String INFRA_RUN_SECURITY_DIR = "infra/run/security/";
- String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt";
-
- String AM_CONFIG_GENERATION = "am.config.generation";
String APP_CONF_DIR = "conf";
- String APP_RESOURCES = "application.resources";
- String APP_RESOURCES_DIR = "app/resources";
-
- String APP_INSTALL_DIR = "app/install";
+ String APP_LIB_DIR = "lib";
String OUT_FILE = "stdout.txt";
String ERR_FILE = "stderr.txt";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 0c249d0..588d330 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -20,7 +20,6 @@ package org.apache.slider.common.tools;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -43,20 +42,15 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
import org.apache.slider.core.persist.Filenames;
-import org.apache.slider.core.persist.InstancePaths;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.FilenameFilter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
-import java.util.Enumeration;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
import static org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
@@ -153,34 +147,6 @@ public class CoreFileSystem {
}
/**
- * Build up the path string for package install location -no attempt to
- * create the directory is made
- *
- * @return the path for persistent app package
- */
- public Path buildPackageDirPath(String packageName, String packageVersion) {
- Preconditions.checkNotNull(packageName);
- Path path = getBaseApplicationPath();
- path = new Path(path, SliderKeys.PACKAGE_DIRECTORY + "/" + packageName);
- if (SliderUtils.isSet(packageVersion)) {
- path = new Path(path, packageVersion);
- }
- return path;
- }
-
- /**
- * Build up the path string for package install location -no attempt to
- * create the directory is made
- *
- * @return the path for persistent app package
- */
- public Path buildClusterSecurityDirPath(String clusterName) {
- Preconditions.checkNotNull(clusterName);
- Path path = buildClusterDirPath(clusterName);
- return new Path(path, SliderKeys.SECURITY_DIR);
- }
-
- /**
* Build up the path string for keytab install location -no attempt to
* create the directory is made
*
@@ -390,36 +356,6 @@ public class CoreFileSystem {
}
/**
- * Create the application-instance specific temporary directory
- * in the DFS
- *
- * @param clustername name of the cluster
- * @param subdir application ID
- * @return the path; this directory will already have been created
- */
- public Path createAppInstanceTempPath(String clustername, String subdir)
- throws IOException {
- Path tmp = getTempPathForCluster(clustername);
- Path instancePath = new Path(tmp, subdir);
- fileSystem.mkdirs(instancePath);
- return instancePath;
- }
-
- /**
- * Create the application-instance specific temporary directory
- * in the DFS
- *
- * @param clustername name of the cluster
- * @return the path; this directory will already have been deleted
- */
- public Path purgeAppInstanceTempFiles(String clustername) throws
- IOException {
- Path tmp = getTempPathForCluster(clustername);
- fileSystem.delete(tmp, true);
- return tmp;
- }
-
- /**
* Get the base path
*
* @return the base path optionally configured by
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index bc8e139..6dc51ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -48,8 +48,6 @@ import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.Slider;
import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
@@ -2540,14 +2538,4 @@ public final class SliderUtils {
long totalMinutes = days * 24 * 60 + hours * 24 + minutes;
return totalMinutes * 60 + seconds;
}
-
- public static void resolve(Application application) {
- org.apache.slider.api.resource.Configuration global = application
- .getConfiguration();
- for (Component component : application.getComponents()) {
- mergeMapsIgnoreDuplicateKeys(component.getConfiguration().getProperties(),
- global.getProperties());
- }
- // TODO merge other information to components
- }
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
deleted file mode 100644
index 3505ac3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.persist;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.SliderKeys;
-
-/**
- * Build up all the paths of an instance relative to the supplied instance
- * directory.
- */
-public class InstancePaths {
-
- public final Path instanceDir;
- public final Path snapshotConfPath;
- public final Path generatedConfPath;
- public final Path historyPath;
- public final Path dataPath;
- public final Path tmpPath;
- public final Path tmpPathAM;
- public final Path appDefPath;
- public final Path addonsPath;
-
- public InstancePaths(Path instanceDir) {
- this.instanceDir = instanceDir;
- snapshotConfPath =
- new Path(instanceDir, SliderKeys.SNAPSHOT_CONF_DIR_NAME);
- generatedConfPath =
- new Path(instanceDir, SliderKeys.GENERATED_CONF_DIR_NAME);
- historyPath = new Path(instanceDir, SliderKeys.HISTORY_DIR_NAME);
- dataPath = new Path(instanceDir, SliderKeys.DATA_DIR_NAME);
- tmpPath = new Path(instanceDir, SliderKeys.TMP_DIR_PREFIX);
- tmpPathAM = new Path(tmpPath, SliderKeys.AM_DIR_PREFIX);
- appDefPath = new Path(tmpPath, SliderKeys.APP_DEF_DIR);
- addonsPath = new Path(tmpPath, SliderKeys.ADDONS_DIR);
- }
-
- @Override
- public String toString() {
- return "instance at " + instanceDir;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index 185dcd4..ea92ff7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -18,8 +18,10 @@
package org.apache.slider.providers;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.ConfigFile;
@@ -30,6 +32,7 @@ import org.codehaus.jettison.json.JSONObject;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -76,12 +79,50 @@ public abstract class AbstractClientProvider {
/**
* Validate the config files.
* @param configFiles config file list
- * @param fileSystem file system
+ * @param fs file system
*/
- public void validateConfigFiles(List<ConfigFile> configFiles, FileSystem
- fileSystem) throws IOException {
- for (ConfigFile configFile : configFiles) {
- validateConfigFile(configFile, fileSystem);
+ public void validateConfigFiles(List<ConfigFile> configFiles,
+ FileSystem fs) throws IOException {
+ Set<String> destFileSet = new HashSet<>();
+
+ for (ConfigFile file : configFiles) {
+ if (file.getType() == null) {
+ throw new IllegalArgumentException("File type is empty");
+ }
+
+ if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
+ .isEmpty(file.getSrcFile())) {
+ throw new IllegalArgumentException(
+ "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
+
+ }
+ if (!StringUtils.isEmpty(file.getSrcFile())) {
+ Path p = new Path(file.getSrcFile());
+ if (!fs.exists(p)) {
+ throw new IllegalArgumentException(
+ "Src_file does not exist for config file: " + file
+ .getSrcFile());
+ }
+ }
+
+ if (StringUtils.isEmpty(file.getDestFile())) {
+ throw new IllegalArgumentException("Dest_file is empty.");
+ }
+
+ if (destFileSet.contains(file.getDestFile())) {
+ throw new IllegalArgumentException(
+ "Duplicated ConfigFile exists: " + file.getDestFile());
+ }
+ destFileSet.add(file.getDestFile());
+
+ java.nio.file.Path destPath = Paths.get(file.getDestFile());
+ if (!destPath.isAbsolute() && destPath.getNameCount() > 1) {
+ throw new IllegalArgumentException("Non-absolute dest_file has more " +
+ "than one path element");
+ }
+
+ // provider-specific validation
+ validateConfigFile(file, fs);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
index 9c52643..5ecc374 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
@@ -22,7 +22,6 @@ import org.apache.slider.api.resource.Artifact;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.providers.docker.DockerProviderFactory;
import org.apache.slider.providers.tarball.TarballProviderFactory;
-import org.apache.slider.util.RestApiErrorMessages;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,7 +29,7 @@ import org.slf4j.LoggerFactory;
* Base class for factories.
*/
public abstract class SliderProviderFactory {
- protected static final Logger log =
+ protected static final Logger LOG =
LoggerFactory.getLogger(SliderProviderFactory.class);
protected SliderProviderFactory() {}
@@ -58,10 +57,10 @@ public abstract class SliderProviderFactory {
public static synchronized SliderProviderFactory createSliderProviderFactory(
Artifact artifact) {
if (artifact == null || artifact.getType() == null) {
- log.info("Loading service provider type default");
+ LOG.debug("Loading service provider type default");
return DefaultProviderFactory.getInstance();
}
- log.info("Loading service provider type {}", artifact.getType());
+ LOG.debug("Loading service provider type {}", artifact.getType());
switch (artifact.getType()) {
// TODO add handling for custom types?
// TODO handle application
@@ -70,8 +69,9 @@ public abstract class SliderProviderFactory {
case TARBALL:
return TarballProviderFactory.getInstance();
default:
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+ throw new IllegalArgumentException(String.format("Resolution error, " +
+ "%s should not be passed to createSliderProviderFactory",
+ artifact.getType()));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java
index 65a55f0..9dd3499 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java
@@ -45,6 +45,6 @@ public class TarballProviderService extends AbstractProviderService {
LocalResourceType type = LocalResourceType.ARCHIVE;
LocalResource packageResource = fileSystem.createAmResource(
artifact, type);
- launcher.addLocalResource(APP_INSTALL_DIR, packageResource);
+ launcher.addLocalResource(APP_LIB_DIR, packageResource);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 84dde08..0c3fcea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -107,7 +107,6 @@ import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
-import org.apache.slider.core.persist.JsonSerDeser;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderService;
@@ -157,7 +156,7 @@ import org.apache.slider.server.services.workflow.ServiceThreadFactory;
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.apache.slider.server.services.workflow.WorkflowRpcService;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.apache.slider.util.ServiceApiUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -389,9 +388,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
private boolean securityEnabled;
private ContentCache contentCache;
- private static final JsonSerDeser<Application> jsonSerDeser =
- new JsonSerDeser<Application>(Application.class,
- PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
/**
* resource limits
@@ -590,9 +586,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
Path appDir = new Path((serviceArgs.getAppDefDir()));
SliderFileSystem fs = getClusterFS();
fs.setAppDir(appDir);
- Path appJson = new Path(appDir, appName + ".json");
- log.info("Loading application definition from " + appJson);
- application = jsonSerDeser.load(fs.getFileSystem(), appJson);
+ application = ServiceApiUtil.loadApplication(fs, appName);
log.info("Application Json: " + application);
stateForProviders.setApplicationName(appName);
Configuration serviceConf = getConfig();
@@ -821,7 +815,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
binding.releaseSelector = new MostRecentContainerReleaseSelector();
binding.nodeReports = nodeReports;
binding.application = application;
- binding.serviceHdfsDir = fs.buildClusterDirPath(appName).toString();
+ binding.serviceHdfsDir = new Path(fs.buildClusterDirPath(appName),
+ SliderKeys.DATA_DIR_NAME).toString();
appState.buildInstance(binding);
// build up environment variables that the AM wants set in every container
@@ -874,11 +869,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
scheduleFailureWindowResets(application.getConfiguration());
scheduleEscalation(application.getConfiguration());
- for (Component component : application.getComponents()) {
- // Merge app-level configuration into component level configuration
- component.getConfiguration().mergeFrom(application.getConfiguration());
- }
-
try {
// schedule YARN Registry registration
queue(new ActionRegisterServiceInstance(appName, appid, application));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
index ac89ed8..676db82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
@@ -21,7 +21,7 @@ public interface RestApiErrorMessages {
String ERROR_APPLICATION_NAME_INVALID =
"Application name is either empty or not provided";
String ERROR_APPLICATION_NAME_INVALID_FORMAT =
- "Application name is not valid - only lower case letters, digits,"
+ "Application name %s is not valid - only lower case letters, digits,"
+ " underscore and hyphen are allowed";
String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
@@ -76,7 +76,7 @@ public interface RestApiErrorMessages {
String ERROR_ABSENT_NUM_OF_INSTANCE =
"Num of instances should appear either globally or per component";
String ERROR_ABSENT_LAUNCH_COMMAND =
- "launch command should appear if type is slider-zip or none";
+ "Launch_command is required when type is not DOCKER";
String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at"
+ " component level, needs corresponding values set at application level";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
index d7c72a3..80a31c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
@@ -25,118 +25,158 @@ import org.apache.hadoop.fs.Path;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.api.resource.Configuration;
import org.apache.slider.api.resource.Resource;
+import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.persist.JsonSerDeser;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.SliderProviderFactory;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class ServiceApiUtil {
- private static final Logger log =
+ private static final Logger LOG =
LoggerFactory.getLogger(ServiceApiUtil.class);
+ private static JsonSerDeser<Application> jsonSerDeser =
+ new JsonSerDeser<>(Application.class,
+ PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+
+ @VisibleForTesting
+ public static void setJsonSerDeser(JsonSerDeser jsd) {
+ jsonSerDeser = jsd;
+ }
+
@VisibleForTesting
- public static void validateApplicationPayload(Application application,
- FileSystem fs) throws IOException {
+ public static void validateAndResolveApplication(Application application,
+ SliderFileSystem fs) throws IOException {
if (StringUtils.isEmpty(application.getName())) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
}
if (!SliderUtils.isClusternameValid(application.getName())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT);
+ throw new IllegalArgumentException(String.format(
+ RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT,
+ application.getName()));
}
// If the application has no components do top-level checks
if (!hasComponent(application)) {
- // artifact
- if (application.getArtifact() == null) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
- }
- if (StringUtils.isEmpty(application.getArtifact().getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
-
- // If artifact is of type APPLICATION, add a slider specific property
- if (application.getArtifact().getType()
- == Artifact.TypeEnum.APPLICATION) {
- if (application.getConfiguration() == null) {
- application.setConfiguration(new Configuration());
+ // If artifact is of type APPLICATION, read other application components
+ if (application.getArtifact() != null && application.getArtifact()
+ .getType() == Artifact.TypeEnum.APPLICATION) {
+ if (StringUtils.isEmpty(application.getArtifact().getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
}
+ Application otherApplication = loadApplication(fs,
+ application.getArtifact().getId());
+ application.setComponents(otherApplication.getComponents());
+ application.setArtifact(null);
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(application.getQuicklinks(),
+ otherApplication.getQuicklinks());
+ } else {
+ // Since it is a simple app with no components, create a default
+ // component
+ Component comp = createDefaultComponent(application);
+ validateComponent(comp, fs.getFileSystem());
+ application.getComponents().add(comp);
+ if (application.getLifetime() == null) {
+ application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
+ }
+ return;
}
- // resource
- validateApplicationResource(application.getResource(), null,
- application.getArtifact().getType());
+ }
- // container size
- if (application.getNumberOfContainers() == null
- || application.getNumberOfContainers() < 0) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID + ": "
- + application.getNumberOfContainers());
+ // Validate there are no component name collisions (collisions are not
+ // currently supported) and add any components from external applications
+ // TODO allow name collisions? see AppState#roles
+ // TODO or add prefix to external component names?
+ Configuration globalConf = application.getConfiguration();
+ Set<String> componentNames = new HashSet<>();
+ List<Component> componentsToRemove = new ArrayList<>();
+ List<Component> componentsToAdd = new ArrayList<>();
+ for (Component comp : application.getComponents()) {
+ if (componentNames.contains(comp.getName())) {
+ throw new IllegalArgumentException("Component name collision: " +
+ comp.getName());
}
- validateConfigFile(application.getConfiguration().getFiles(), fs);
- // Since it is a simple app with no components, create a default component
- application.getComponents().add(createDefaultComponent(application));
- } else {
- // If the application has components, then run checks for each component.
- // Let global values take effect if component level values are not
- // provided.
- Artifact globalArtifact = application.getArtifact();
- Resource globalResource = application.getResource();
- Long globalNumberOfContainers = application.getNumberOfContainers();
- for (Component comp : application.getComponents()) {
- // artifact
- if (comp.getArtifact() == null) {
- comp.setArtifact(globalArtifact);
- }
- // If still null raise validation exception
- if (comp.getArtifact() == null) {
- throw new IllegalArgumentException(String
- .format(RestApiErrorMessages.ERROR_ARTIFACT_FOR_COMP_INVALID,
- comp.getName()));
- }
+ // If artifact is of type APPLICATION (which cannot be filled from
+ // global), read external application and add its components to this
+ // application
+ if (comp.getArtifact() != null && comp.getArtifact().getType() ==
+ Artifact.TypeEnum.APPLICATION) {
if (StringUtils.isEmpty(comp.getArtifact().getId())) {
- throw new IllegalArgumentException(String
- .format(RestApiErrorMessages.ERROR_ARTIFACT_ID_FOR_COMP_INVALID,
- comp.getName()));
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
}
-
- // If artifact is of type APPLICATION, add a slider specific property
- if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) {
- if (comp.getConfiguration() == null) {
- comp.setConfiguration(new Configuration());
+ LOG.info("Marking {} for removal", comp.getName());
+ componentsToRemove.add(comp);
+ List<Component> externalComponents = getApplicationComponents(fs,
+ comp.getArtifact().getId());
+ for (Component c : externalComponents) {
+ Component override = application.getComponent(c.getName());
+ if (override != null && override.getArtifact() == null) {
+ // allow properties from external components to be overridden /
+ // augmented by properties in this component, except for artifact
+ // which must be read from external component
+ override.mergeFrom(c);
+ LOG.info("Merging external component {} from external {}", c
+ .getName(), comp.getName());
+ } else {
+ if (componentNames.contains(c.getName())) {
+ throw new IllegalArgumentException("Component name collision: " +
+ c.getName());
+ }
+ componentNames.add(c.getName());
+ componentsToAdd.add(c);
+ LOG.info("Adding component {} from external {}", c.getName(),
+ comp.getName());
}
- comp.setName(comp.getArtifact().getId());
- }
-
- // resource
- if (comp.getResource() == null) {
- comp.setResource(globalResource);
}
- validateApplicationResource(comp.getResource(), comp,
- comp.getArtifact().getType());
+ } else {
+ // otherwise handle as a normal component
+ componentNames.add(comp.getName());
+ // configuration
+ comp.getConfiguration().mergeFrom(globalConf);
+ }
+ }
+ application.getComponents().removeAll(componentsToRemove);
+ application.getComponents().addAll(componentsToAdd);
- // container count
- if (comp.getNumberOfContainers() == null) {
- comp.setNumberOfContainers(globalNumberOfContainers);
- }
- if (comp.getNumberOfContainers() == null
- || comp.getNumberOfContainers() < 0) {
- throw new IllegalArgumentException(String.format(
- RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID
- + ": " + comp.getNumberOfContainers(), comp.getName()));
- }
- validateConfigFile(comp.getConfiguration().getFiles(), fs);
+ // Validate components and let global values take effect if component level
+ // values are not provided
+ Artifact globalArtifact = application.getArtifact();
+ Resource globalResource = application.getResource();
+ Long globalNumberOfContainers = application.getNumberOfContainers();
+ String globalLaunchCommand = application.getLaunchCommand();
+ for (Component comp : application.getComponents()) {
+ // fill in global artifact unless it is type APPLICATION
+ if (comp.getArtifact() == null && application.getArtifact() != null
+ && application.getArtifact().getType() != Artifact.TypeEnum
+ .APPLICATION) {
+ comp.setArtifact(globalArtifact);
+ }
+ // fill in global resource
+ if (comp.getResource() == null) {
+ comp.setResource(globalResource);
}
+ // fill in global container count
+ if (comp.getNumberOfContainers() == null) {
+ comp.setNumberOfContainers(globalNumberOfContainers);
+ }
+ // fill in global launch command
+ if (comp.getLaunchCommand() == null) {
+ comp.setLaunchCommand(globalLaunchCommand);
+ }
+ validateComponent(comp, fs.getFileSystem());
}
// Application lifetime if not specified, is set to unlimited lifetime
@@ -145,52 +185,54 @@ public class ServiceApiUtil {
}
}
- // 1) Verify the src_file exists and non-empty for template
- // 2) dest_file is absolute path
- private static void validateConfigFile(List<ConfigFile> list, FileSystem fs)
+ public static void validateComponent(Component comp, FileSystem fs)
throws IOException {
- Set<String> destFileSet = new HashSet<>();
+ AbstractClientProvider compClientProvider = SliderProviderFactory
+ .getClientProvider(comp.getArtifact());
+ compClientProvider.validateArtifact(comp.getArtifact(), fs);
- for (ConfigFile file : list) {
- if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
- .isEmpty(file.getSrcFile())) {
- throw new IllegalArgumentException(
- "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
+ if (comp.getLaunchCommand() == null && (comp.getArtifact() == null || comp
+ .getArtifact().getType() != Artifact.TypeEnum.DOCKER)) {
+ throw new IllegalArgumentException(RestApiErrorMessages
+ .ERROR_ABSENT_LAUNCH_COMMAND);
+ }
- }
- if (!StringUtils.isEmpty(file.getSrcFile())) {
- Path p = new Path(file.getSrcFile());
- if (!fs.exists(p)) {
- throw new IllegalArgumentException(
- "Src_file does not exist for config file: " + file
- .getSrcFile());
- }
- }
+ validateApplicationResource(comp.getResource(), comp);
- if (StringUtils.isEmpty(file.getDestFile())) {
- throw new IllegalArgumentException("Dest_file is empty.");
- }
- // validate dest_file is absolute
- if (!Paths.get(file.getDestFile()).isAbsolute()) {
- throw new IllegalArgumentException(
- "Dest_file must be absolute path: " + file.getDestFile());
- }
-
- if (destFileSet.contains(file.getDestFile())) {
- throw new IllegalArgumentException(
- "Duplicated ConfigFile exists: " + file.getDestFile());
- }
- destFileSet.add(file.getDestFile());
+ if (comp.getNumberOfContainers() == null
+ || comp.getNumberOfContainers() < 0) {
+ throw new IllegalArgumentException(String.format(
+ RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID
+ + ": " + comp.getNumberOfContainers(), comp.getName()));
}
+ compClientProvider.validateConfigFiles(comp.getConfiguration()
+ .getFiles(), fs);
+ }
+
+ @VisibleForTesting
+ public static List<Component> getApplicationComponents(SliderFileSystem
+ fs, String appName) throws IOException {
+ return loadApplication(fs, appName).getComponents();
}
+ public static Application loadApplication(SliderFileSystem fs, String
+ appName) throws IOException {
+ Path appJson = getAppJsonPath(fs, appName);
+ LOG.info("Loading application definition from " + appJson);
+ Application externalApplication = jsonSerDeser.load(fs.getFileSystem(),
+ appJson);
+ return externalApplication;
+ }
+
+ public static Path getAppJsonPath(SliderFileSystem fs, String appName) {
+ Path appDir = fs.buildClusterDirPath(appName);
+ Path appJson = new Path(appDir, appName + ".json");
+ return appJson;
+ }
private static void validateApplicationResource(Resource resource,
- Component comp, Artifact.TypeEnum artifactType) {
+ Component comp) {
// Only apps/components of type APPLICATION can skip resource requirement
- if (resource == null && artifactType == Artifact.TypeEnum.APPLICATION) {
- return;
- }
if (resource == null) {
throw new IllegalArgumentException(
comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String
@@ -255,6 +297,7 @@ public class ServiceApiUtil {
comp.setResource(app.getResource());
comp.setNumberOfContainers(app.getNumberOfContainers());
comp.setLaunchCommand(app.getLaunchCommand());
+ comp.setConfiguration(app.getConfiguration());
return comp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb984564/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
index 07d8c10..59ccda7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
@@ -34,6 +34,7 @@ import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.main.ServiceLauncher;
import org.apache.slider.utils.SliderTestBase;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -50,6 +51,7 @@ import java.util.UUID;
public class TestKeytabCommandOptions extends SliderTestBase {
private static SliderFileSystem testFileSystem;
+ private File testFolderDir;
@Before
public void setupFilesystem() throws IOException {
@@ -57,11 +59,18 @@ public class TestKeytabCommandOptions extends SliderTestBase {
YarnConfiguration configuration = SliderUtils.createConfiguration();
fileSystem.setConf(configuration);
testFileSystem = new SliderFileSystem(fileSystem, configuration);
- File testFolderDir = new File(testFileSystem
+ testFolderDir = new File(testFileSystem
.buildKeytabInstallationDirPath("").toUri().getPath());
FileUtils.deleteDirectory(testFolderDir);
}
+ @After
+ public void cleanup() throws IOException {
+ if (testFolderDir != null && testFolderDir.exists()) {
+ FileUtils.deleteDirectory(testFolderDir);
+ }
+ }
+
@Test
public void testInstallKeytab() throws Throwable {
// create a mock keytab file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org