You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/06/21 18:33:27 UTC
[05/50] [abbrv] hadoop git commit: YARN-6255. Refactor
yarn-native-services framework. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68ec5e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
new file mode 100644
index 0000000..7e8cf5b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.util;
+
+public interface RestApiConstants {
+ String CONTEXT_ROOT = "/services/v1";
+ String APPLICATIONS_API_RESOURCE_PATH = "/applications";
+ String CONTAINERS_API_RESOURCE_PATH = "/containers";
+ String SLIDER_APPMASTER_COMPONENT_NAME = "slider-appmaster";
+ String SLIDER_CONFIG_SCHEMA = "http://example.org/specification/v2.0.0";
+ String METAINFO_SCHEMA_VERSION = "2.1";
+ String COMPONENT_TYPE_YARN_DOCKER = "yarn_docker";
+
+ String DEFAULT_START_CMD = "/bootstrap/privileged-centos6-sshd";
+ String DEFAULT_COMPONENT_NAME = "DEFAULT";
+ String DEFAULT_IMAGE = "centos:centos6";
+ String DEFAULT_NETWORK = "bridge";
+ String DEFAULT_COMMAND_PATH = "/usr/bin/docker";
+ String DEFAULT_USE_NETWORK_SCRIPT = "yes";
+
+ String PLACEHOLDER_APP_NAME = "${APP_NAME}";
+ String PLACEHOLDER_APP_COMPONENT_NAME = "${APP_COMPONENT_NAME}";
+ String PLACEHOLDER_COMPONENT_ID = "${COMPONENT_ID}";
+
+ String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
+ String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
+ String PROPERTY_APP_LIFETIME = "docker.lifetime";
+ String PROPERTY_APP_RUNAS_USER = "APP_RUNAS_USER";
+ Long DEFAULT_UNLIMITED_LIFETIME = -1l;
+
+ Integer HTTP_STATUS_CODE_ACCEPTED = 202;
+ String ARTIFACT_TYPE_SLIDER_ZIP = "slider-zip";
+
+ Integer GET_APPLICATIONS_THREAD_POOL_SIZE = 200;
+
+ String PROPERTY_PYTHON_PATH = "python.path";
+ String PROPERTY_DNS_DEPENDENCY = "site.global.dns.dependency";
+
+ String COMMAND_ORDER_SUFFIX_START = "-START";
+ String COMMAND_ORDER_SUFFIX_STARTED = "-STARTED";
+ String EXPORT_GROUP_NAME = "QuickLinks";
+
+ Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
+ Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002;
+ Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003;
+ Integer ERROR_CODE_APP_NAME_INVALID = 404004;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68ec5e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
new file mode 100644
index 0000000..0f6247d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.util;
+
+public interface RestApiErrorMessages {
+ String ERROR_APPLICATION_NAME_INVALID =
+ "Application name is either empty or not provided";
+ String ERROR_APPLICATION_NAME_INVALID_FORMAT =
+ "Application name is not valid - only lower case letters, digits,"
+ + " underscore and hyphen are allowed";
+
+ String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
+ String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found";
+ String ERROR_APPLICATION_IN_USE = "Application already exists in started"
+ + " state";
+ String ERROR_APPLICATION_INSTANCE_EXISTS = "Application already exists in"
+ + " stopped/failed state (either restart with PUT or destroy with DELETE"
+ + " before creating a new one)";
+
+ String ERROR_SUFFIX_FOR_COMPONENT =
+ " for component %s (nor at the global level)";
+ String ERROR_ARTIFACT_INVALID = "Artifact is not provided";
+ String ERROR_ARTIFACT_FOR_COMP_INVALID =
+ ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_ARTIFACT_ID_INVALID =
+ "Artifact id (like docker image name) is either empty or not provided";
+ String ERROR_ARTIFACT_ID_FOR_COMP_INVALID =
+ ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+
+ String ERROR_RESOURCE_INVALID = "Resource is not provided";
+ String ERROR_RESOURCE_FOR_COMP_INVALID =
+ ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_MEMORY_INVALID =
+ "Application resource or memory not provided";
+ String ERROR_RESOURCE_CPUS_INVALID =
+ "Application resource or cpus not provided";
+ String ERROR_RESOURCE_CPUS_INVALID_RANGE =
+ "Unacceptable no of cpus specified, either zero or negative";
+ String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID =
+ ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID =
+ ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE =
+ ERROR_RESOURCE_CPUS_INVALID_RANGE
+ + " for component %s (or at the global level)";
+ String ERROR_CONTAINERS_COUNT_INVALID =
+ "Required no of containers not specified";
+ String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
+ ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+
+ String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED =
+ "Cannot specify" + " cpus/memory along with profile";
+ String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED =
+ ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED
+ + " for component %s";
+ String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET =
+ "Resource profile is not " + "supported yet. Please specify cpus/memory.";
+
+ String ERROR_NULL_ARTIFACT_ID =
+ "Artifact Id can not be null if artifact type is none";
+ String ERROR_ABSENT_NUM_OF_INSTANCE =
+ "Num of instances should appear either globally or per component";
+ String ERROR_ABSENT_LAUNCH_COMMAND =
+ "launch command should appear if type is slider-zip or none";
+
+ String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at"
+ + " component level, needs corresponding values set at application level";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68ec5e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
new file mode 100644
index 0000000..776ce00
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.Configuration;
+import org.apache.slider.api.resource.Resource;
+import org.apache.slider.common.tools.SliderUtils;
+
+public class ServiceApiUtil {
+
+ @VisibleForTesting
+ public static void validateApplicationPostPayload(Application application) {
+ if (StringUtils.isEmpty(application.getName())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
+ }
+ if (!SliderUtils.isClusternameValid(application.getName())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT);
+ }
+
+ // If the application has no components do top-level checks
+ if (!hasComponent(application)) {
+ // artifact
+ if (application.getArtifact() == null) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+ }
+ if (StringUtils.isEmpty(application.getArtifact().getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+
+ // If artifact is of type APPLICATION, add a slider specific property
+ if (application.getArtifact().getType()
+ == Artifact.TypeEnum.APPLICATION) {
+ if (application.getConfiguration() == null) {
+ application.setConfiguration(new Configuration());
+ }
+ }
+ // resource
+ validateApplicationResource(application.getResource(), null,
+ application.getArtifact().getType());
+
+ // container size
+ if (application.getNumberOfContainers() == null) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID);
+ }
+
+ // Since it is a simple app with no components, create a default component
+ application.getComponents().add(createDefaultComponent(application));
+ } else {
+ // If the application has components, then run checks for each component.
+ // Let global values take effect if component level values are not
+ // provided.
+ Artifact globalArtifact = application.getArtifact();
+ Resource globalResource = application.getResource();
+ Long globalNumberOfContainers = application.getNumberOfContainers();
+ for (Component comp : application.getComponents()) {
+ // artifact
+ if (comp.getArtifact() == null) {
+ comp.setArtifact(globalArtifact);
+ }
+ // If still null raise validation exception
+ if (comp.getArtifact() == null) {
+ throw new IllegalArgumentException(String
+ .format(RestApiErrorMessages.ERROR_ARTIFACT_FOR_COMP_INVALID,
+ comp.getName()));
+ }
+ if (StringUtils.isEmpty(comp.getArtifact().getId())) {
+ throw new IllegalArgumentException(String
+ .format(RestApiErrorMessages.ERROR_ARTIFACT_ID_FOR_COMP_INVALID,
+ comp.getName()));
+ }
+
+ // If artifact is of type APPLICATION, add a slider specific property
+ if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) {
+ if (comp.getConfiguration() == null) {
+ comp.setConfiguration(new Configuration());
+ }
+ comp.setName(comp.getArtifact().getId());
+ }
+
+ // resource
+ if (comp.getResource() == null) {
+ comp.setResource(globalResource);
+ }
+ validateApplicationResource(comp.getResource(), comp,
+ comp.getArtifact().getType());
+
+ // container count
+ if (comp.getNumberOfContainers() == null) {
+ comp.setNumberOfContainers(globalNumberOfContainers);
+ }
+ if (comp.getNumberOfContainers() == null) {
+ throw new IllegalArgumentException(String.format(
+ RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID,
+ comp.getName()));
+ }
+ }
+ }
+
+ // Application lifetime if not specified, is set to unlimited lifetime
+ if (application.getLifetime() == null) {
+ application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
+ }
+ }
+
+ private static void validateApplicationResource(Resource resource,
+ Component comp, Artifact.TypeEnum artifactType) {
+ // Only apps/components of type APPLICATION can skip resource requirement
+ if (resource == null && artifactType == Artifact.TypeEnum.APPLICATION) {
+ return;
+ }
+ if (resource == null) {
+ throw new IllegalArgumentException(
+ comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String
+ .format(RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
+ comp.getName()));
+ }
+ // One and only one of profile OR cpus & memory can be specified. Specifying
+ // both raises validation error.
+ if (StringUtils.isNotEmpty(resource.getProfile()) && (
+ resource.getCpus() != null || StringUtils
+ .isNotEmpty(resource.getMemory()))) {
+ throw new IllegalArgumentException(comp == null ?
+ RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED :
+ String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
+ comp.getName()));
+ }
+ // Currently resource profile is not supported yet, so we will raise
+ // validation error if only resource profile is specified
+ if (StringUtils.isNotEmpty(resource.getProfile())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET);
+ }
+
+ String memory = resource.getMemory();
+ Integer cpus = resource.getCpus();
+ if (StringUtils.isEmpty(memory)) {
+ throw new IllegalArgumentException(
+ comp == null ? RestApiErrorMessages.ERROR_RESOURCE_MEMORY_INVALID :
+ String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
+ comp.getName()));
+ }
+ if (cpus == null) {
+ throw new IllegalArgumentException(
+ comp == null ? RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID :
+ String.format(
+ RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID,
+ comp.getName()));
+ }
+ if (cpus <= 0) {
+ throw new IllegalArgumentException(comp == null ?
+ RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID_RANGE : String
+ .format(
+ RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
+ comp.getName()));
+ }
+ }
+
+ public static boolean hasComponent(Application application) {
+ if (application.getComponents() == null || application.getComponents()
+ .isEmpty()) {
+ return false;
+ }
+ return true;
+ }
+
+ public static Component createDefaultComponent(Application app) {
+ Component comp = new Component();
+ comp.setName(RestApiConstants.DEFAULT_COMPONENT_NAME);
+ comp.setArtifact(app.getArtifact());
+ comp.setResource(app.getResource());
+ comp.setNumberOfContainers(app.getNumberOfContainers());
+ comp.setLaunchCommand(app.getLaunchCommand());
+ return comp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68ec5e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
index b8bdc59..bfcab23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
@@ -80,22 +80,14 @@ message UpgradeContainersRequestProto {
message UpgradeContainersResponseProto {
}
-/**
- * flex the cluster
- */
-message FlexClusterRequestProto {
- required string clusterSpec = 1;
+message FlexComponentRequestProto {
+ optional string name = 1;
+ optional int32 numberOfContainers = 2;
}
-
-/**
- * flex the cluster
- */
-message FlexClusterResponseProto {
- required bool response = 1;
+message FlexComponentResponseProto {
}
-
/**
* void request
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68ec5e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
index d68da2b..8a0faf9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
@@ -61,24 +61,14 @@ service SliderClusterProtocolPB {
rpc upgradeContainers(UpgradeContainersRequestProto)
returns(UpgradeContainersResponseProto);
- /**
- * Flex the cluster.
- */
- rpc flexCluster(FlexClusterRequestProto)
- returns(FlexClusterResponseProto);
+ rpc flexComponent(FlexComponentRequestProto) returns (FlexComponentResponseProto);
/**
* Get the current cluster status
*/
rpc getJSONClusterStatus(GetJSONClusterStatusRequestProto)
returns(GetJSONClusterStatusResponseProto);
-
- /**
- * Get the instance definition
- */
- rpc getInstanceDefinition(GetInstanceDefinitionRequestProto)
- returns(GetInstanceDefinitionResponseProto);
-
+
/**
* List all running nodes in a role
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68ec5e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
index a2517d5..96bfe0f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
@@ -24,10 +24,6 @@
<value>true</value>
</property>
<property>
- <name>slider.provider.agent</name>
- <value>org.apache.slider.providers.agent.AgentProviderFactory</value>
- </property>
- <property>
<name>slider.provider.docker</name>
<value>org.apache.slider.providers.docker.DockerProviderFactory</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68ec5e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
deleted file mode 100644
index b955931..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import java.lang.reflect.Method;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.LogAggregationContext;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.client.SliderYarnClientImpl;
-import org.apache.slider.common.SliderKeys;
-import org.easymock.EasyMock;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestAppMasterLauncher {
- SliderYarnClientImpl mockYarnClient;
- YarnClientApplication yarnClientApp;
- ApplicationSubmissionContext appSubmissionContext;
- Set<String> tags = Collections.emptySet();
- AppMasterLauncher appMasterLauncher = null;
- boolean isOldApi = true;
- Method rolledLogsIncludeMethod = null;
- Method rolledLogsExcludeMethod = null;
-
- @Before
- public void initialize() throws Exception {
- mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class);
- yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class);
- appSubmissionContext = EasyMock
- .createNiceMock(ApplicationSubmissionContext.class);
- EasyMock.expect(yarnClientApp.getApplicationSubmissionContext())
- .andReturn(appSubmissionContext).once();
- EasyMock.expect(mockYarnClient.createApplication())
- .andReturn(yarnClientApp).once();
-
- try {
- LogAggregationContext.class.getMethod("newInstance", String.class,
- String.class, String.class, String.class);
- isOldApi = false;
- rolledLogsIncludeMethod = LogAggregationContext.class
- .getMethod("getRolledLogsIncludePattern");
- rolledLogsExcludeMethod = LogAggregationContext.class
- .getMethod("getRolledLogsExcludePattern");
- } catch (Exception e) {
- isOldApi = true;
- }
- }
-
- /**
- * These tests will probably fail when compiled against hadoop 2.7+. Please
- * refer to SLIDER-810. It has been purposely not modified so that it fails
- * and that someone needs to modify the code in
- * {@code AbstractLauncher#extractLogAggregationContext(Map)}. Comments are
- * provided in that method as to what needs to be done.
- *
- * @throws Exception
- */
- @Test
- public void testExtractLogAggregationContext() throws Exception {
- Map<String, String> options = new HashMap<String, String>();
- options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS,
- " | slider*.txt |agent.out| |");
- options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
- "command*.json| agent.log* | ");
-
- EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
- appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
- null, mockYarnClient, false, null, options, tags, null);
-
- // Verify the include/exclude patterns
- String expectedInclude = "slider*.txt|agent.out";
- String expectedExclude = "command*.json|agent.log*";
- assertPatterns(expectedInclude, expectedExclude);
-
- EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
-
- }
-
- @Test
- public void testExtractLogAggregationContextEmptyIncludePattern()
- throws Exception {
- Map<String, String> options = new HashMap<String, String>();
- options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " ");
- options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
- "command*.json| agent.log* | ");
-
- EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
- appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
- null, mockYarnClient, false, null, options, tags, null);
-
- // Verify the include/exclude patterns
- String expectedInclude = isOldApi ? "" : ".*";
- String expectedExclude = "command*.json|agent.log*";
- assertPatterns(expectedInclude, expectedExclude);
-
- EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
- }
-
- @Test
- public void testExtractLogAggregationContextEmptyIncludeAndExcludePattern()
- throws Exception {
- Map<String, String> options = new HashMap<String, String>();
- options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "");
- options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, " ");
-
- EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
- appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
- null, mockYarnClient, false, null, options, tags, null);
-
- // Verify the include/exclude patterns
- String expectedInclude = isOldApi ? "" : ".*";
- String expectedExclude = "";
- assertPatterns(expectedInclude, expectedExclude);
-
- EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
- }
-
- private void assertPatterns(String expectedIncludePattern,
- String expectedExcludePattern) throws Exception {
- if (isOldApi) {
- Assert.assertEquals(expectedIncludePattern,
- appMasterLauncher.logAggregationContext.getIncludePattern());
- Assert.assertEquals(expectedExcludePattern,
- appMasterLauncher.logAggregationContext.getExcludePattern());
- } else {
- Assert.assertEquals(expectedIncludePattern,
- (String) rolledLogsIncludeMethod
- .invoke(appMasterLauncher.logAggregationContext));
- Assert.assertEquals(expectedExcludePattern,
- (String) rolledLogsExcludeMethod
- .invoke(appMasterLauncher.logAggregationContext));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68ec5e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
deleted file mode 100644
index a8f6b26..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.client.SliderYarnClientImpl;
-import org.apache.slider.common.SliderKeys;
-import org.easymock.EasyMock;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestAppMasterLauncherWithAmReset {
- SliderYarnClientImpl mockYarnClient;
- YarnClientApplication yarnClientApp;
- ApplicationSubmissionContext appSubmissionContext;
- GetNewApplicationResponse newApp;
- Set<String> tags = Collections.emptySet();
- AppMasterLauncher appMasterLauncher = null;
- boolean isOldApi = true;
-
- @Before
- public void initialize() throws Exception {
- mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class);
- yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class);
- newApp = EasyMock.createNiceMock(GetNewApplicationResponse.class);
- EasyMock.expect(mockYarnClient.createApplication())
- .andReturn(new YarnClientApplication(newApp,
- Records.newRecord(ApplicationSubmissionContext.class)));
- }
-
- @Test
- public void testExtractYarnResourceManagerAmRetryCountWindowMs() throws
- Exception {
- Map<String, String> options = new HashMap<String, String>();
- final String expectedInterval = Integer.toString (120000);
- options.put(ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS,
- expectedInterval);
- EasyMock.replay(mockYarnClient, yarnClientApp);
-
- appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null,
- null, mockYarnClient, false, null, options, tags, null);
-
- ApplicationSubmissionContext ctx = appMasterLauncher.application
- .getApplicationSubmissionContext();
- String retryIntervalWindow = Long.toString(ctx
- .getAttemptFailuresValidityInterval());
- Assert.assertEquals(expectedInterval, retryIntervalWindow);
- }
-
- @Test
- public void testExtractYarnResourceManagerAmRetryCountWindowMsDefaultValue()
- throws Exception {
- Map<String, String> options = new HashMap<String, String>();
- EasyMock.replay(mockYarnClient, yarnClientApp);
-
- appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null,
- null, mockYarnClient, false, null, options, tags, null);
-
- ApplicationSubmissionContext ctx = appMasterLauncher.application
- .getApplicationSubmissionContext();
- long retryIntervalWindow = ctx.getAttemptFailuresValidityInterval();
- Assert.assertEquals(ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS,
- retryIntervalWindow);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68ec5e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java
deleted file mode 100644
index a1986cd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster;
-
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.core.conf.MapOperations;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- *
- */
-public class TestServiceRecordAttributes extends Assert {
-
- @Test
- public void testAppConfigProvidedServiceRecordAttributes() throws Exception {
- Map<String, String> options = new HashMap<>();
- options.put("slider.some.arbitrary.option", "arbitrary value");
- options.put("service.record.attribute.one_attribute", "one_attribute_value");
- options.put("service.record.attribute.second_attribute", "second_attribute_value");
- MapOperations serviceProps = new MapOperations(SliderKeys.COMPONENT_AM, options);
- options = new HashMap<>();
- options.put("some.component.attribute", "component_attribute_value");
- options.put("service.record.attribute.component_attribute", "component_attribute_value");
- MapOperations compProps = new MapOperations("TEST_COMP", options);
-
- SliderAppMaster appMaster = new SliderAppMaster();
-
- ServiceRecord appServiceRecord = new ServiceRecord();
-
- appMaster.setProvidedServiceRecordAttributes(serviceProps, appServiceRecord);
-
- assertNull("property should not be attribute",
- appServiceRecord.get("slider.some.arbitrary.option"));
- assertEquals("wrong value", "one_attribute_value",
- appServiceRecord.get("one_attribute"));
- assertEquals("wrong value", "second_attribute_value",
- appServiceRecord.get("second_attribute"));
-
- ServiceRecord compServiceRecord = new ServiceRecord();
-
- appMaster.setProvidedServiceRecordAttributes(compProps, compServiceRecord);
-
- assertNull("should not be attribute",
- compServiceRecord.get("some.component.attribute"));
- assertEquals("wrong value", "component_attribute_value",
- compServiceRecord.get("component_attribute"));
-
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org