You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/09/25 23:38:46 UTC
[47/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
deleted file mode 100644
index 1fa07ce..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.net.HostAndPort;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class ZookeeperUtils {
- public static final int DEFAULT_PORT = 2181;
-
- public static String buildConnectionString(String zkHosts, int port) {
- String zkPort = Integer.toString(port);
- //parse the hosts
- String[] hostlist = zkHosts.split(",", 0);
- String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
- return quorum;
- }
-
- /**
- * Take a quorum list and split it to (trimmed) pairs
- * @param hostPortQuorumList list of form h1:port, h2:port2,...
- * @return a possibly empty list of values between commas. They may not be
- * valid hostname:port pairs
- */
- public static List<String> splitToPairs(String hostPortQuorumList) {
- // split an address hot
- String[] strings = StringUtils.getStrings(hostPortQuorumList);
- int len = 0;
- if (strings != null) {
- len = strings.length;
- }
- List<String> tuples = new ArrayList<String>(len);
- if (strings != null) {
- for (String s : strings) {
- tuples.add(s.trim());
- }
- }
- return tuples;
- }
-
- /**
- * Split a quorum list into a list of hostnames and ports
- * @param hostPortQuorumList split to a list of hosts and ports
- * @return a list of values
- */
- public static List<HostAndPort> splitToHostsAndPorts(String hostPortQuorumList) {
- // split an address hot
- String[] strings = StringUtils.getStrings(hostPortQuorumList);
- int len = 0;
- if (strings != null) {
- len = strings.length;
- }
- List<HostAndPort> list = new ArrayList<HostAndPort>(len);
- if (strings != null) {
- for (String s : strings) {
- list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
- }
- }
- return list;
- }
-
- /**
- * Build up to a hosts only list
- * @param hostAndPorts
- * @return a list of the hosts only
- */
- public static String buildHostsOnlyList(List<HostAndPort> hostAndPorts) {
- StringBuilder sb = new StringBuilder();
- for (HostAndPort hostAndPort : hostAndPorts) {
- sb.append(hostAndPort.getHostText()).append(",");
- }
- if (sb.length() > 0) {
- sb.delete(sb.length() - 1, sb.length());
- }
- return sb.toString();
- }
-
- public static String buildQuorumEntry(HostAndPort hostAndPort,
- int defaultPort) {
- String s = hostAndPort.toString();
- if (hostAndPort.hasPort()) {
- return s;
- } else {
- return s + ":" + defaultPort;
- }
- }
-
- /**
- * Build a quorum list, injecting a ":defaultPort" ref if needed on
- * any entry without one
- * @param hostAndPorts
- * @param defaultPort
- * @return
- */
- public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) {
- List<String> entries = new ArrayList<String>(hostAndPorts.size());
- for (HostAndPort hostAndPort : hostAndPorts) {
- entries.add(buildQuorumEntry(hostAndPort, defaultPort));
- }
- return SliderUtils.join(entries, ",", false);
- }
-
- public static String convertToHostsOnlyList(String quorum) throws
- BadConfigException {
- List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
- return ZookeeperUtils.buildHostsOnlyList(hostAndPorts);
- }
-
- public static List<HostAndPort> splitToHostsAndPortsStrictly(String quorum) throws
- BadConfigException {
- List<HostAndPort> hostAndPorts =
- ZookeeperUtils.splitToHostsAndPorts(quorum);
- if (hostAndPorts.isEmpty()) {
- throw new BadConfigException("empty zookeeper quorum");
- }
- return hostAndPorts;
- }
-
- public static int getFirstPort(String quorum, int defVal) throws
- BadConfigException {
- List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
- int port = hostAndPorts.get(0).getPortOrDefault(defVal);
- return port;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto
deleted file mode 100644
index 0a21c24..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.hadoop.yarn.proto";
-option java_outer_classname = "ClientAMProtocol";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.yarn;
-
-service ClientAMProtocolService {
- rpc flexComponents(FlexComponentsRequestProto) returns (FlexComponentsResponseProto);
- rpc getStatus(GetStatusRequestProto) returns (GetStatusResponseProto);
- rpc stop(StopRequestProto) returns (StopResponseProto);
-}
-
-message FlexComponentsRequestProto {
- repeated ComponentCountProto components = 1;
-}
-
-message ComponentCountProto {
- optional string name = 1;
- optional int64 numberOfContainers = 2;
-}
-
-message FlexComponentsResponseProto{
-}
-
-message GetStatusRequestProto {
-
-}
-message GetStatusResponseProto {
- optional string status = 1;
-}
-
-message StopRequestProto {
-
-}
-
-message StopResponseProto {
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
deleted file mode 100644
index 4fa81ee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import com.google.common.base.Supplier;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.NMClient;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
-import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.service.component.ComponentState;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-
-import static org.mockito.Mockito.mock;
-
-public class MockServiceAM extends ServiceMaster {
-
- Application application;
- // The list of containers fed by tests to be returned on
- // AMRMClientCallBackHandler#onContainersAllocated
- final List<Container> feedContainers =
- Collections.synchronizedList(new LinkedList<>());
-
- public MockServiceAM(Application application) {
- super(application.getName());
- this.application = application;
- }
-
-
- @Override
- protected ContainerId getAMContainerId()
- throws BadClusterStateException {
- return ContainerId.newContainerId(ApplicationAttemptId
- .newInstance(ApplicationId.fromString(application.getId()), 1), 1);
- }
-
- @Override
- protected Path getAppDir() {
- Path path = new Path(new Path("target", "apps"), application.getName());
- System.out.println("Application path: " + path);
- return path;
- }
-
- @Override
- protected ServiceScheduler createServiceScheduler(ServiceContext context)
- throws IOException, YarnException {
- return new ServiceScheduler(context) {
-
- @Override
- protected YarnRegistryViewForProviders createYarnRegistryOperations(
- ServiceContext context, RegistryOperations registryClient) {
- return mock(YarnRegistryViewForProviders.class);
- }
-
- @Override
- protected AMRMClientAsync<AMRMClient.ContainerRequest> createAMRMClient() {
- AMRMClientImpl client1 = new AMRMClientImpl() {
- @Override public AllocateResponse allocate(float progressIndicator)
- throws YarnException, IOException {
-
- AllocateResponse.AllocateResponseBuilder builder =
- AllocateResponse.newBuilder();
- synchronized (feedContainers) {
- if (feedContainers.isEmpty()) {
- System.out.println("Allocating........ no containers");
- return builder.build();
- } else {
- // The AMRMClient will return containers for compoenent that are
- // at FLEXING state
- List<Container> allocatedContainers = new LinkedList<>();
- Iterator<Container> itor = feedContainers.iterator();
- while (itor.hasNext()) {
- Container c = itor.next();
- org.apache.hadoop.yarn.service.component.Component component =
- componentsById.get(c.getAllocationRequestId());
- if (component.getState() == ComponentState.FLEXING) {
- System.out.println("Allocated container " + c.getId());
- allocatedContainers.add(c);
- itor.remove();
- }
- }
- return builder.allocatedContainers(allocatedContainers).build();
- }
- }
- }
-
- @Override
- public RegisterApplicationMasterResponse registerApplicationMaster(
- String appHostName, int appHostPort, String appTrackingUrl) {
- return mock(RegisterApplicationMasterResponse.class);
- }
-
- @Override public void unregisterApplicationMaster(
- FinalApplicationStatus appStatus, String appMessage,
- String appTrackingUrl) {
- // DO nothing
- }
- };
-
- return AMRMClientAsync
- .createAMRMClientAsync(client1, 1000,
- this.new AMRMClientCallback());
- }
-
- @Override
- public NMClientAsync createNMClient() {
- NMClientAsync nmClientAsync = super.createNMClient();
- nmClientAsync.setClient(mock(NMClient.class));
- return nmClientAsync;
- }
- };
- }
-
- @Override protected void loadApplicationJson(ServiceContext context,
- SliderFileSystem fs) throws IOException {
- context.application = application;
- }
-
- /**
- *
- * @param application The application for the component
- * @param id The id for the container
- * @param compName The component to which the container is fed
- * @return
- */
- public Container feedContainerToComp(Application application, int id,
- String compName) {
- ApplicationId applicationId = ApplicationId.fromString(application.getId());
- ContainerId containerId = ContainerId
- .newContainerId(ApplicationAttemptId.newInstance(applicationId, 1), id);
- NodeId nodeId = NodeId.newInstance("localhost", 1234);
- Container container = Container
- .newInstance(containerId, nodeId, "localhost",
- Resource.newInstance(100, 1), Priority.newInstance(0), null);
-
- long allocateId =
- context.scheduler.getAllComponents().get(compName).getAllocateId();
- container.setAllocationRequestId(allocateId);
- synchronized (feedContainers) {
- feedContainers.add(container);
- }
- return container;
- }
-
- public void flexComponent(String compName, long numberOfContainers)
- throws IOException {
- ClientAMProtocol.ComponentCountProto componentCountProto =
- ClientAMProtocol.ComponentCountProto.newBuilder().setName(compName)
- .setNumberOfContainers(numberOfContainers).build();
- ClientAMProtocol.FlexComponentsRequestProto requestProto =
- ClientAMProtocol.FlexComponentsRequestProto.newBuilder()
- .addComponents(componentCountProto).build();
- context.clientAMService.flexComponents(requestProto);
- }
-
- public Component getComponent(String compName) {
- return context.scheduler.getAllComponents().get(compName);
- }
-
- public void waitForDependenciesSatisfied(String compName)
- throws TimeoutException, InterruptedException {
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override public Boolean get() {
- return context.scheduler.getAllComponents().get(compName)
- .areDependenciesReady();
- }
- }, 1000, 20000);
- }
-
- public void waitForNumDesiredContainers(String compName,
- int numDesiredContainers) throws TimeoutException, InterruptedException {
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override public Boolean get() {
- return context.scheduler.getAllComponents().get(compName)
- .getNumDesiredInstances() == numDesiredContainers;
- }
- }, 1000, 20000);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
deleted file mode 100644
index 73172bf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
-
-public class ServiceTestUtils {
-
- public static final JsonSerDeser<Application> JSON_SER_DESER =
- new JsonSerDeser<>(Application.class,
- PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
-
- // Example service definition
- // 2 components, each of which has 2 containers.
- protected Application createExampleApplication() {
- Application exampleApp = new Application();
- exampleApp.setName("example-app");
- exampleApp.addComponent(createComponent("compa"));
- exampleApp.addComponent(createComponent("compb"));
- return exampleApp;
- }
-
- protected Component createComponent(String name) {
- return createComponent(name, 2L, "sleep 1000");
- }
-
- protected Component createComponent(String name, long numContainers,
- String command) {
- Component comp1 = new Component();
- comp1.setNumberOfContainers(numContainers);
- comp1.setLaunchCommand(command);
- comp1.setName(name);
- Resource resource = new Resource();
- comp1.setResource(resource);
- resource.setMemory("128");
- resource.setCpus(1);
- return comp1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
deleted file mode 100644
index 1a22875..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
+++ /dev/null
@@ -1,529 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
-import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
-import static org.easymock.EasyMock.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * Test for ServiceApiUtil helper methods.
- */
-public class TestServiceApiUtil {
- private static final Logger LOG = LoggerFactory
- .getLogger(TestServiceApiUtil.class);
- private static final String EXCEPTION_PREFIX = "Should have thrown " +
- "exception: ";
- private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
- "exception: ";
-
- private static final String LEN_64_STR =
- "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01";
-
- private static final YarnConfiguration CONF_DEFAULT_DNS = new
- YarnConfiguration();
- private static final YarnConfiguration CONF_DNS_ENABLED = new
- YarnConfiguration();
-
- @BeforeClass
- public static void init() {
- CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
- }
-
- @Test(timeout = 90000)
- public void testResourceValidation() throws Exception {
- assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
- .length());
-
- SliderFileSystem sfs = initMock(null);
-
- Application app = new Application();
-
- // no name
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no name");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
- }
-
- // bad format name
- String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR};
- for (String badName : badNames) {
- app.setName(badName);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- ERROR_APPLICATION_NAME_INVALID_FORMAT, badName), e.getMessage());
- }
- }
-
- // launch command not specified
- app.setName(LEN_64_STR);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DEFAULT_DNS);
- Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
- } catch (IllegalArgumentException e) {
- assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
- e.getMessage());
- }
-
- // launch command not specified
- app.setName(LEN_64_STR.substring(0, RegistryConstants
- .MAX_FQDN_LABEL_LENGTH));
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
- } catch (IllegalArgumentException e) {
- assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
- e.getMessage());
- }
-
- // resource not specified
- app.setLaunchCommand("sleep 3600");
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no resource");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
- DEFAULT_COMPONENT_NAME), e.getMessage());
- }
-
- // memory not specified
- Resource res = new Resource();
- app.setResource(res);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no memory");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
- DEFAULT_COMPONENT_NAME), e.getMessage());
- }
-
- // invalid no of cpus
- res.setMemory("100mb");
- res.setCpus(-2);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(
- EXCEPTION_PREFIX + "application with invalid no of cpus");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
- DEFAULT_COMPONENT_NAME), e.getMessage());
- }
-
- // number of containers not specified
- res.setCpus(2);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no container count");
- } catch (IllegalArgumentException e) {
- Assert.assertTrue(e.getMessage()
- .contains(ERROR_CONTAINERS_COUNT_INVALID));
- }
-
- // specifying profile along with cpus/memory raises exception
- res.setProfile("hbase_finance_large");
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX
- + "application with resource profile along with cpus/memory");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(RestApiErrorMessages
- .ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
- DEFAULT_COMPONENT_NAME),
- e.getMessage());
- }
-
- // currently resource profile alone is not supported.
- // TODO: remove the next test once resource profile alone is supported.
- res.setCpus(null);
- res.setMemory(null);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with resource profile only");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
- e.getMessage());
- }
-
- // unset profile here and add cpus/memory back
- res.setProfile(null);
- res.setCpus(2);
- res.setMemory("2gb");
-
- // null number of containers
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "null number of containers");
- } catch (IllegalArgumentException e) {
- Assert.assertTrue(e.getMessage()
- .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
- }
-
- // negative number of containers
- app.setNumberOfContainers(-1L);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "negative number of containers");
- } catch (IllegalArgumentException e) {
- Assert.assertTrue(e.getMessage()
- .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
- }
-
- // everything valid here
- app.setNumberOfContainers(5L);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- LOG.error("application attributes specified should be valid here", e);
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
- }
-
- @Test
- public void testArtifacts() throws IOException {
- SliderFileSystem sfs = initMock(null);
-
- Application app = new Application();
- app.setName("name");
- Resource res = new Resource();
- app.setResource(res);
- res.setMemory("512M");
- app.setNumberOfContainers(3L);
-
- // no artifact id fails with default type
- Artifact artifact = new Artifact();
- app.setArtifact(artifact);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
- }
-
- // no artifact id fails with APPLICATION type
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
- }
-
- // no artifact id fails with TARBALL type
- artifact.setType(Artifact.TypeEnum.TARBALL);
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
- } catch (IllegalArgumentException e) {
- assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
- }
-
- // everything valid here
- artifact.setType(Artifact.TypeEnum.DOCKER);
- artifact.setId("docker.io/centos:centos7");
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- LOG.error("application attributes specified should be valid here", e);
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- // defaults assigned
- assertEquals(app.getComponents().get(0).getName(),
- DEFAULT_COMPONENT_NAME);
- assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
- }
-
- private static Resource createValidResource() {
- Resource res = new Resource();
- res.setMemory("512M");
- return res;
- }
-
- private static Component createValidComponent(String compName) {
- Component comp = new Component();
- comp.setName(compName);
- comp.setResource(createValidResource());
- comp.setNumberOfContainers(1L);
- return comp;
- }
-
- private static Application createValidApplication(String compName) {
- Application app = new Application();
- app.setLaunchCommand("sleep 3600");
- app.setName("name");
- app.setResource(createValidResource());
- app.setNumberOfContainers(1L);
- if (compName != null) {
- app.addComponent(createValidComponent(compName));
- }
- return app;
- }
-
- private static SliderFileSystem initMock(Application ext) throws IOException {
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- JsonSerDeser<Application> jsonSerDeser = createNiceMock(JsonSerDeser
- .class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- if (ext != null) {
- expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
- .anyTimes();
- }
- replay(sfs, mockFs, jsonSerDeser);
- ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
- return sfs;
- }
-
- @Test
- public void testExternalApplication() throws IOException {
- Application ext = createValidApplication("comp1");
- SliderFileSystem sfs = initMock(ext);
-
- Application app = createValidApplication(null);
-
- Artifact artifact = new Artifact();
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- artifact.setId("id");
- app.setArtifact(artifact);
-
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- assertEquals(1, app.getComponents().size());
- assertNotNull(app.getComponent("comp1"));
- }
-
- @Test
- public void testDuplicateComponents() throws IOException {
- SliderFileSystem sfs = initMock(null);
-
- String compName = "comp1";
- Application app = createValidApplication(compName);
- app.addComponent(createValidComponent(compName));
-
- // duplicate component name fails
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with component collision");
- } catch (IllegalArgumentException e) {
- assertEquals("Component name collision: " + compName, e.getMessage());
- }
- }
-
- @Test
- public void testExternalDuplicateComponent() throws IOException {
- Application ext = createValidApplication("comp1");
- SliderFileSystem sfs = initMock(ext);
-
- Application app = createValidApplication("comp1");
- Artifact artifact = new Artifact();
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- artifact.setId("id");
- app.getComponent("comp1").setArtifact(artifact);
-
- // duplicate component name okay in the case of APPLICATION component
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
- }
-
- @Test
- public void testExternalComponent() throws IOException {
- Application ext = createValidApplication("comp1");
- SliderFileSystem sfs = initMock(ext);
-
- Application app = createValidApplication("comp2");
- Artifact artifact = new Artifact();
- artifact.setType(Artifact.TypeEnum.APPLICATION);
- artifact.setId("id");
- app.setArtifact(artifact);
-
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- assertEquals(1, app.getComponents().size());
- // artifact ID not inherited from global
- assertNotNull(app.getComponent("comp2"));
-
- // set APPLICATION artifact id on component
- app.getComponent("comp2").setArtifact(artifact);
-
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- assertEquals(1, app.getComponents().size());
- // original component replaced by external component
- assertNotNull(app.getComponent("comp1"));
- }
-
- public static void verifyDependencySorting(List<Component> components,
- Component... expectedSorting) {
- Collection<Component> actualSorting = ServiceApiUtil.sortByDependencies(
- components);
- assertEquals(expectedSorting.length, actualSorting.size());
- int i = 0;
- for (Component component : actualSorting) {
- assertEquals(expectedSorting[i++], component);
- }
- }
-
- @Test
- public void testDependencySorting() throws IOException {
- Component a = new Component().name("a");
- Component b = new Component().name("b");
- Component c = new Component().name("c");
- Component d = new Component().name("d").dependencies(Arrays.asList("c"));
- Component e = new Component().name("e").dependencies(Arrays.asList("b",
- "d"));
-
- verifyDependencySorting(Arrays.asList(a, b, c), a, b, c);
- verifyDependencySorting(Arrays.asList(c, a, b), c, a, b);
- verifyDependencySorting(Arrays.asList(a, b, c, d, e), a, b, c, d, e);
- verifyDependencySorting(Arrays.asList(e, d, c, b, a), c, b, a, d, e);
-
- c.setDependencies(Arrays.asList("e"));
- try {
- verifyDependencySorting(Arrays.asList(a, b, c, d, e));
- Assert.fail(EXCEPTION_PREFIX + "components with dependency cycle");
- } catch (IllegalArgumentException ex) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_DEPENDENCY_CYCLE, Arrays.asList(c, d,
- e)), ex.getMessage());
- }
-
- SliderFileSystem sfs = initMock(null);
- Application application = createValidApplication(null);
- application.setComponents(Arrays.asList(c, d, e));
- try {
- ServiceApiUtil.validateAndResolveApplication(application, sfs,
- CONF_DEFAULT_DNS);
- Assert.fail(EXCEPTION_PREFIX + "components with bad dependencies");
- } catch (IllegalArgumentException ex) {
- assertEquals(String.format(
- RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, "b", "e"), ex
- .getMessage());
- }
- }
-
- @Test
- public void testInvalidComponent() throws IOException {
- SliderFileSystem sfs = initMock(null);
- testComponent(sfs);
- }
-
- @Test
- public void testValidateCompName() {
- String[] invalidNames = {
- "EXAMPLE", // UPPER case not allowed
- "example_app" // underscore not allowed.
- };
- for (String name : invalidNames) {
- try {
- ServiceApiUtil.validateCompName(name);
- Assert.fail();
- } catch (IllegalArgumentException ex) {
- ex.printStackTrace();
- }
- }
- }
-
- private static void testComponent(SliderFileSystem sfs)
- throws IOException {
- int maxLen = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
- assertEquals(19, Long.toString(Long.MAX_VALUE).length());
- maxLen = maxLen - Long.toString(Long.MAX_VALUE).length();
-
- String compName = LEN_64_STR.substring(0, maxLen + 1);
- Application app = createValidApplication(null);
- app.addComponent(createValidComponent(compName));
-
- // invalid component name fails if dns is enabled
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- Assert.fail(EXCEPTION_PREFIX + "application with invalid component name");
- } catch (IllegalArgumentException e) {
- assertEquals(String.format(RestApiErrorMessages
- .ERROR_COMPONENT_NAME_INVALID, maxLen, compName), e.getMessage());
- }
-
- // does not fail if dns is disabled
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DEFAULT_DNS);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
-
- compName = LEN_64_STR.substring(0, maxLen);
- app = createValidApplication(null);
- app.addComponent(createValidComponent(compName));
-
- // does not fail
- try {
- ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
- } catch (IllegalArgumentException e) {
- Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
deleted file mode 100644
index a36e0b4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ /dev/null
@@ -1,472 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.curator.test.TestingCluster;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
-import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM;
-import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-
-/**
- * End to end tests to test deploying services with MiniYarnCluster and a in-JVM
- * ZK testing cluster.
- */
-public class TestYarnNativeServices extends ServiceTestUtils{
-
- private static final Log LOG =
- LogFactory.getLog(TestYarnNativeServices.class);
-
- private MiniYARNCluster yarnCluster = null;
- private MiniDFSCluster hdfsCluster = null;
- private FileSystem fs = null;
- protected Configuration conf = null;
- private static final int NUM_NMS = 1;
- private File basedir;
-
- @Rule
- public TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @Before
- public void setup() throws Exception {
- setupInternal(NUM_NMS);
- }
-
- private void setupInternal(int numNodeManager)
- throws Exception {
- LOG.info("Starting up YARN cluster");
-// Logger rootLogger = LogManager.getRootLogger();
-// rootLogger.setLevel(Level.DEBUG);
- conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
- // reduce the teardown waiting time
- conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);
- conf.set("yarn.log.dir", "target");
- // mark if we need to launch the v1 timeline server
- // disable aux-service based timeline aggregators
- conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
- conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
- // Enable ContainersMonitorImpl
- conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
- LinuxResourceCalculatorPlugin.class.getName());
- conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
- ProcfsBasedProcessTree.class.getName());
- conf.setBoolean(
- YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true);
- conf.setBoolean(TIMELINE_SERVICE_ENABLED, false);
- conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100);
- conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000);
- conf.setLong(AM_RESOURCE_MEM, 526);
- conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5);
- // Disable vmem check to disallow NM killing the container
- conf.setBoolean(NM_VMEM_CHECK_ENABLED, false);
- conf.setBoolean(NM_PMEM_CHECK_ENABLED, false);
- // setup zk cluster
- TestingCluster zkCluster;
- zkCluster = new TestingCluster(1);
- zkCluster.start();
- conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString());
- conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
- LOG.info("ZK cluster: " + zkCluster.getConnectString());
-
- fs = FileSystem.get(conf);
- basedir = new File("target", "apps");
- if (basedir.exists()) {
- FileUtils.deleteDirectory(basedir);
- } else {
- basedir.mkdirs();
- }
-
- conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
-
- if (yarnCluster == null) {
- yarnCluster =
- new MiniYARNCluster(TestYarnNativeServices.class.getSimpleName(), 1,
- numNodeManager, 1, 1);
- yarnCluster.init(conf);
- yarnCluster.start();
-
- waitForNMsToRegister();
-
- URL url = Thread.currentThread().getContextClassLoader()
- .getResource("yarn-site.xml");
- if (url == null) {
- throw new RuntimeException(
- "Could not find 'yarn-site.xml' dummy file in classpath");
- }
- Configuration yarnClusterConfig = yarnCluster.getConfig();
- yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- new File(url.getPath()).getParent());
- //write the document to a buffer (not directly to the file, as that
- //can cause the file being written to get read -which will then fail.
- ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
- yarnClusterConfig.writeXml(bytesOut);
- bytesOut.close();
- //write the bytes to the file in the classpath
- OutputStream os = new FileOutputStream(new File(url.getPath()));
- os.write(bytesOut.toByteArray());
- os.close();
- LOG.info("Write yarn-site.xml configs to: " + url);
- }
- if (hdfsCluster == null) {
- HdfsConfiguration hdfsConfig = new HdfsConfiguration();
- hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
- .numDataNodes(1).build();
- }
-
- try {
- Thread.sleep(2000);
- } catch (InterruptedException e) {
- LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
- }
-
-
- }
-
- private void waitForNMsToRegister() throws Exception {
- int sec = 60;
- while (sec >= 0) {
- if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size()
- >= NUM_NMS) {
- break;
- }
- Thread.sleep(1000);
- sec--;
- }
- }
-
- @After
- public void tearDown() throws IOException {
- if (yarnCluster != null) {
- try {
- yarnCluster.stop();
- } finally {
- yarnCluster = null;
- }
- }
- if (hdfsCluster != null) {
- try {
- hdfsCluster.shutdown();
- } finally {
- hdfsCluster = null;
- }
- }
- if (basedir != null) {
- FileUtils.deleteDirectory(basedir);
- }
- SliderFileSystem sfs = new SliderFileSystem(conf);
- Path appDir = sfs.getBaseApplicationPath();
- sfs.getFileSystem().delete(appDir, true);
- }
-
-
-
- // End-to-end test to use ServiceClient to deploy a service.
- // 1. Create a service with 2 components, each of which has 2 containers
- // 2. Flex up each component to 3 containers and check the component instance names
- // 3. Flex down each component to 1 container and check the component instance names
- // 4. Flex up each component to 2 containers and check the component instance names
- // 5. Stop the service
- // 6. Destroy the service
- @Test (timeout = 200000)
- public void testCreateFlexStopDestroyService() throws Exception {
- ServiceClient client = createClient();
- Application exampleApp = createExampleApplication();
- client.actionCreate(exampleApp);
- SliderFileSystem fileSystem = new SliderFileSystem(conf);
- Path appDir = fileSystem.buildClusterDirPath(exampleApp.getName());
- // check app.json is persisted.
- Assert.assertTrue(
- fs.exists(new Path(appDir, exampleApp.getName() + ".json")));
- waitForAllCompToBeReady(client, exampleApp);
-
- // Flex two components, each from 2 container to 3 containers.
- flexComponents(client, exampleApp, 3L);
- // wait for flex to be completed, increase from 2 to 3 containers.
- waitForAllCompToBeReady(client, exampleApp);
- // check all instances name for each component are in sequential order.
- checkCompInstancesInOrder(client, exampleApp);
-
- // flex down to 1
- flexComponents(client, exampleApp, 1L);
- waitForAllCompToBeReady(client, exampleApp);
- checkCompInstancesInOrder(client, exampleApp);
-
- // check component dir and registry are cleaned up.
-
- // flex up again to 2
- flexComponents(client, exampleApp, 2L);
- waitForAllCompToBeReady(client, exampleApp);
- checkCompInstancesInOrder(client, exampleApp);
-
- // stop the service
- LOG.info("Stop the service");
- client.actionStop(exampleApp.getName(), true);
- ApplicationReport report = client.getYarnClient()
- .getApplicationReport(ApplicationId.fromString(exampleApp.getId()));
- // AM unregisters with RM successfully
- Assert.assertEquals(FINISHED, report.getYarnApplicationState());
- Assert.assertEquals(FinalApplicationStatus.ENDED,
- report.getFinalApplicationStatus());
-
- LOG.info("Destroy the service");
- //destroy the service and check the app dir is deleted from fs.
- client.actionDestroy(exampleApp.getName());
- // check the application dir on hdfs (in this case, local fs) are deleted.
- Assert.assertFalse(fs.exists(appDir));
- }
-
- // Create compa with 2 containers
- // Create compb with 2 containers which depends on compa
- // Check containers for compa started before containers for compb
- @Test (timeout = 200000)
- public void testComponentStartOrder() throws Exception {
- ServiceClient client = createClient();
- Application exampleApp = new Application();
- exampleApp.setName("teststartorder");
- exampleApp.addComponent(createComponent("compa", 2, "sleep 1000"));
- Component compb = createComponent("compb", 2, "sleep 1000");
-
- // Let compb depedends on compa;
- compb.setDependencies(Collections.singletonList("compa"));
- exampleApp.addComponent(compb);
-
- client.actionCreate(exampleApp);
- waitForAllCompToBeReady(client, exampleApp);
-
- // check that containers for compa are launched before containers for compb
- checkContainerLaunchDependencies(client, exampleApp, "compa", "compb");
-
- client.actionStop(exampleApp.getName(), true);
- client.actionDestroy(exampleApp.getName());
- }
-
- // Check containers launched are in dependency order
- // Get all containers into a list and sort based on container launch time e.g.
- // compa-c1, compa-c2, compb-c1, compb-c2;
- // check that the container's launch time are align with the dependencies.
- private void checkContainerLaunchDependencies(ServiceClient client,
- Application exampleApp, String... compOrder)
- throws IOException, YarnException {
- Application retrievedApp = client.getStatus(exampleApp.getName());
- List<Container> containerList = new ArrayList<>();
- for (Component component : retrievedApp.getComponents()) {
- containerList.addAll(component.getContainers());
- }
- // sort based on launchTime
- containerList
- .sort((o1, o2) -> o1.getLaunchTime().compareTo(o2.getLaunchTime()));
- LOG.info("containerList: " + containerList);
- // check the containers are in the dependency order.
- int index = 0;
- for (String comp : compOrder) {
- long num = retrievedApp.getComponent(comp).getNumberOfContainers();
- for (int i = 0; i < num; i++) {
- String compInstanceName = containerList.get(index).getComponentName();
- String compName =
- compInstanceName.substring(0, compInstanceName.lastIndexOf('-'));
- Assert.assertEquals(comp, compName);
- index++;
- }
- }
- }
-
-
- private Map<String, Long> flexComponents(ServiceClient client,
- Application exampleApp, long count) throws YarnException, IOException {
- Map<String, Long> compCounts = new HashMap<>();
- compCounts.put("compa", count);
- compCounts.put("compb", count);
- // flex will update the persisted conf to reflect latest number of containers.
- exampleApp.getComponent("compa").setNumberOfContainers(count);
- exampleApp.getComponent("compb").setNumberOfContainers(count);
- client.flexByRestService(exampleApp.getName(), compCounts);
- return compCounts;
- }
-
- // Check each component's comp instances name are in sequential order.
- // E.g. If there are two instances compA-1 and compA-2
- // When flex up to 4 instances, it should be compA-1 , compA-2, compA-3, compA-4
- // When flex down to 3 instances, it should be compA-1 , compA-2, compA-3.
- private void checkCompInstancesInOrder(ServiceClient client,
- Application exampleApp) throws IOException, YarnException {
- Application application = client.getStatus(exampleApp.getName());
- for (Component comp : application.getComponents()) {
- checkEachCompInstancesInOrder(comp);
- }
- }
-
- private void checkRegistryAndCompDirDeleted() {
-
- }
-
- private void checkEachCompInstancesInOrder(Component component) {
- long expectedNumInstances = component.getNumberOfContainers();
- Assert.assertEquals(expectedNumInstances, component.getContainers().size());
- TreeSet<String> instances = new TreeSet<>();
- for (Container container : component.getContainers()) {
- instances.add(container.getComponentName());
- }
-
- int i = 0;
- for (String s : instances) {
- Assert.assertEquals(component.getName() + "-" + i, s);
- i++;
- }
- }
-
- private void waitForOneCompToBeReady(ServiceClient client,
- Application exampleApp, String readyComp)
- throws TimeoutException, InterruptedException {
- long numExpectedContainers =
- exampleApp.getComponent(readyComp).getNumberOfContainers();
- GenericTestUtils.waitFor(() -> {
- try {
- Application retrievedApp = client.getStatus(exampleApp.getName());
- Component retrievedComp = retrievedApp.getComponent(readyComp);
-
- if (retrievedComp.getContainers() != null
- && retrievedComp.getContainers().size() == numExpectedContainers) {
- LOG.info(readyComp + " found " + numExpectedContainers
- + " containers running");
- return true;
- } else {
- LOG.info(" Waiting for " + readyComp + "'s containers to be running");
- return false;
- }
- } catch (Exception e) {
- e.printStackTrace();
- return false;
- }
- }, 2000, 200000);
- }
-
- // wait until all the containers for all components become ready state
- private void waitForAllCompToBeReady(ServiceClient client,
- Application exampleApp) throws TimeoutException, InterruptedException {
- int expectedTotalContainers = countTotalContainers(exampleApp);
- GenericTestUtils.waitFor(() -> {
- try {
- Application retrievedApp = client.getStatus(exampleApp.getName());
- int totalReadyContainers = 0;
- LOG.info("Num Components " + retrievedApp.getComponents().size());
- for (Component component : retrievedApp.getComponents()) {
- LOG.info("looking for " + component.getName());
- LOG.info(component);
- if (component.getContainers() != null) {
- if (component.getContainers().size() == exampleApp
- .getComponent(component.getName()).getNumberOfContainers()) {
- for (Container container : component.getContainers()) {
- LOG.info(
- "Container state " + container.getState() + ", component "
- + component.getName());
- if (container.getState() == ContainerState.READY) {
- totalReadyContainers++;
- LOG.info("Found 1 ready container " + container.getId());
- }
- }
- } else {
- LOG.info(component.getName() + " Expected number of containers "
- + exampleApp.getComponent(component.getName())
- .getNumberOfContainers() + ", current = " + component
- .getContainers());
- }
- }
- }
- LOG.info("Exit loop, totalReadyContainers= " + totalReadyContainers
- + " expected = " + expectedTotalContainers);
- return totalReadyContainers == expectedTotalContainers;
- } catch (Exception e) {
- e.printStackTrace();
- return false;
- }
- }, 2000, 200000);
- }
-
- private ServiceClient createClient() throws Exception {
- ServiceClient client = new ServiceClient() {
- @Override protected Path addJarResource(String appName,
- Map<String, LocalResource> localResources)
- throws IOException, SliderException {
- // do nothing, the Unit test will use local jars
- return null;
- }
- };
- client.init(conf);
- client.start();
- return client;
- }
-
-
- private int countTotalContainers(Application application) {
- int totalContainers = 0;
- for (Component component : application.getComponents()) {
- totalContainers += component.getNumberOfContainers();
- }
- return totalContainers;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
deleted file mode 100644
index a22c000..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-
-/**
- * Test for building / resolving components of type APPLICATION.
- */
-public class TestBuildExternalComponents {
-
- protected Configuration conf = new YarnConfiguration();
- private File basedir;
-
- // Check component names match with expected
- private static void checkComponentNames(List<Component> components,
- Set<String> expectedComponents) {
- Assert.assertEquals(expectedComponents.size(), components.size());
- for (Component comp : components) {
- Assert.assertTrue(expectedComponents.contains(comp.getName()));
- }
- }
-
- // 1. Build the appDef and store on fs
- // 2. check component names
- private void buildAndCheckComponents(String appName, String appDef,
- SliderFileSystem sfs, Set<String> names) throws Throwable {
- String[] args =
- { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
- ClientArgs clientArgs = new ClientArgs(args);
- clientArgs.parse();
- ServiceCLI cli = new ServiceCLI() {
- @Override protected void createServiceClient() {
- client = new ServiceClient();
- client.init(conf);
- client.start();
- }
- };
- cli.exec(clientArgs);
-
- // verify generated conf
- List<Component> components =
- ServiceApiUtil.getApplicationComponents(sfs, appName);
- checkComponentNames(components, names);
- }
-
- @Before
- public void setup() throws IOException {
- basedir = new File("target", "apps");
- if (basedir.exists()) {
- FileUtils.deleteDirectory(basedir);
- } else {
- basedir.mkdirs();
- }
- conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
- }
-
- @After
- public void tearDown() throws IOException {
- if (basedir != null) {
- FileUtils.deleteDirectory(basedir);
- }
- }
-
- // Test applications defining external components(APPLICATION type)
- // can be resolved correctly
- @Test
- public void testExternalComponentBuild() throws Throwable {
- SliderFileSystem sfs = new SliderFileSystem(conf);
-
- Set<String> nameSet = new HashSet<>();
- nameSet.add("simple");
- nameSet.add("master");
- nameSet.add("worker");
-
- // app-1 has 3 components: simple, master, worker
- buildAndCheckComponents("app-1", ExampleAppJson.APP_JSON, sfs, nameSet);
- buildAndCheckComponents("external-0", ExampleAppJson.EXTERNAL_JSON_0, sfs,
- nameSet);
-
- nameSet.add("other");
-
- // external1 has 3 components: simple(APPLICATION - app1), master and other
- buildAndCheckComponents("external-1", ExampleAppJson.EXTERNAL_JSON_1, sfs,
- nameSet);
-
- nameSet.add("another");
-
- // external2 has 2 components: ext(APPLICATION - external1), another
- buildAndCheckComponents("external-2", ExampleAppJson.EXTERNAL_JSON_2, sfs,
- nameSet);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
deleted file mode 100644
index 20c06ab..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-import static org.mockito.Mockito.mock;
-
-public class TestServiceCLI {
-
- protected Configuration conf = new YarnConfiguration();
- private File basedir;
- private ServiceCLI cli;
- private SliderFileSystem fs;
-
- private void buildApp(String appName, String appDef) throws Throwable {
- String[] args =
- { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
- ClientArgs clientArgs = new ClientArgs(args);
- clientArgs.parse();
- cli.exec(clientArgs);
- }
-
- @Before
- public void setup() throws Throwable {
- basedir = new File("target", "apps");
- conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
- fs = new SliderFileSystem(conf);
- if (basedir.exists()) {
- FileUtils.deleteDirectory(basedir);
- } else {
- basedir.mkdirs();
- }
-
- // create a CLI and skip connection to AM
- cli = new ServiceCLI() {
- @Override protected void createServiceClient() {
- client = new ServiceClient() {
- @Override protected ClientAMProtocol getAMProxy(String appName,
- ApplicationReport report) throws IOException {
- return mock(ClientAMProtocol.class);
- }
- @Override protected ClientAMProtocol getAMProxy(String appName)
- throws IOException, YarnException {
- return mock(ClientAMProtocol.class);
- }
- };
- client.init(conf);
- client.start();
- }
- };
- }
-
- @After
- public void tearDown() throws IOException {
- if (basedir != null) {
- FileUtils.deleteDirectory(basedir);
- }
- }
-
- // Test flex components count are persisted.
- @Test
- public void testFlexComponents() throws Throwable {
- buildApp("service-1", ExampleAppJson.APP_JSON);
-
- checkCompCount("master", 1L);
-
- // increase by 2
- String[] flexUpArgs = {"flex", "service-1", "--component", "master" , "+2"};
- ClientArgs clientArgs = new ClientArgs(flexUpArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
-
- checkCompCount("master", 3L);
-
- // decrease by 1
- String[] flexDownArgs = {"flex", "service-1", "--component", "master", "-1"};
- clientArgs = new ClientArgs(flexDownArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
-
- checkCompCount("master", 2L);
-
- String[] flexAbsoluteArgs = {"flex", "service-1", "--component", "master", "10"};
- clientArgs = new ClientArgs(flexAbsoluteArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
-
- checkCompCount("master", 10L);
- }
-
- private void checkCompCount(String compName, long count) throws IOException {
- List<Component> components =
- ServiceApiUtil.getApplicationComponents(fs, "service-1");
- for (Component component : components) {
- if (component.getName().equals(compName)) {
- Assert.assertEquals(count, component.getNumberOfContainers().longValue());
- return;
- }
- }
- Assert.fail();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
deleted file mode 100644
index 9e13200..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-
-import org.apache.hadoop.yarn.service.api.records.Application;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
-
-/**
- * Names of the example configs.
- */
-public final class ExampleAppJson {
-
- public static final String APP_JSON = "app.json";
- public static final String OVERRIDE_JSON = "app-override.json";
- public static final String DEFAULT_JSON = "default.json";
- public static final String EXTERNAL_JSON_0 = "external0.json";
- public static final String EXTERNAL_JSON_1 = "external1.json";
- public static final String EXTERNAL_JSON_2 = "external2.json";
-
- public static final String PACKAGE = "/org/apache/hadoop/yarn/service/conf/examples/";
-
-
- private static final String[] ALL_EXAMPLES = {APP_JSON, OVERRIDE_JSON,
- DEFAULT_JSON};
-
- public static final List<String> ALL_EXAMPLE_RESOURCES = new ArrayList<>();
- static {
- for (String example : ALL_EXAMPLES) {
- ALL_EXAMPLE_RESOURCES.add(PACKAGE + example);
- }
- }
-
- private ExampleAppJson() {
- }
-
- public static Application loadResource(String name) throws IOException {
- return JSON_SER_DESER.fromResource(PACKAGE + name);
- }
-
- public static String resourceName(String name) {
- return "target/test-classes" + PACKAGE + name;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
deleted file mode 100644
index 954d117..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
-import static org.easymock.EasyMock.*;
-
-/**
- * Test global configuration resolution.
- */
-public class TestAppJsonResolve extends Assert {
- protected static final Logger LOG =
- LoggerFactory.getLogger(TestAppJsonResolve.class);
-
- @Test
- public void testOverride() throws Throwable {
- Application orig = ExampleAppJson.loadResource(OVERRIDE_JSON);
-
- Configuration global = orig.getConfiguration();
- assertEquals("a", global.getProperty("g1"));
- assertEquals("b", global.getProperty("g2"));
- assertEquals(2, global.getFiles().size());
-
- Configuration simple = orig.getComponent("simple").getConfiguration();
- assertEquals(0, simple.getProperties().size());
- assertEquals(1, simple.getFiles().size());
-
- Configuration master = orig.getComponent("master").getConfiguration();
- assertEquals("m", master.getProperty("name"));
- assertEquals("overridden", master.getProperty("g1"));
- assertEquals(0, master.getFiles().size());
-
- Configuration worker = orig.getComponent("worker").getConfiguration();
- LOG.info("worker = {}", worker);
- assertEquals(3, worker.getProperties().size());
- assertEquals(0, worker.getFiles().size());
-
- assertEquals("worker", worker.getProperty("name"));
- assertEquals("overridden-by-worker", worker.getProperty("g1"));
- assertNull(worker.getProperty("g2"));
- assertEquals("1000", worker.getProperty("timeout"));
-
- // here is the resolution
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- replay(sfs, mockFs);
- ServiceApiUtil.validateAndResolveApplication(orig, sfs, new
- YarnConfiguration());
-
- global = orig.getConfiguration();
- LOG.info("global = {}", global);
- assertEquals("a", global.getProperty("g1"));
- assertEquals("b", global.getProperty("g2"));
- assertEquals(2, global.getFiles().size());
-
- simple = orig.getComponent("simple").getConfiguration();
- assertEquals(2, simple.getProperties().size());
- assertEquals("a", simple.getProperty("g1"));
- assertEquals("b", simple.getProperty("g2"));
- assertEquals(2, simple.getFiles().size());
-
- Set<ConfigFile> files = new HashSet<>();
- Map<String, String> props = new HashMap<>();
- props.put("k1", "overridden");
- props.put("k2", "v2");
- files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
- .PROPERTIES).props(props));
- files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
- .XML).props(Collections.singletonMap("k3", "v3")));
- assertTrue(files.contains(simple.getFiles().get(0)));
- assertTrue(files.contains(simple.getFiles().get(1)));
-
- master = orig.getComponent("master").getConfiguration();
- LOG.info("master = {}", master);
- assertEquals(3, master.getProperties().size());
- assertEquals("m", master.getProperty("name"));
- assertEquals("overridden", master.getProperty("g1"));
- assertEquals("b", master.getProperty("g2"));
- assertEquals(2, master.getFiles().size());
-
- props.put("k1", "v1");
- files.clear();
- files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
- .PROPERTIES).props(props));
- files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
- .XML).props(Collections.singletonMap("k3", "v3")));
-
- assertTrue(files.contains(master.getFiles().get(0)));
- assertTrue(files.contains(master.getFiles().get(1)));
-
- worker = orig.getComponent("worker").getConfiguration();
- LOG.info("worker = {}", worker);
- assertEquals(4, worker.getProperties().size());
-
- assertEquals("worker", worker.getProperty("name"));
- assertEquals("overridden-by-worker", worker.getProperty("g1"));
- assertEquals("b", worker.getProperty("g2"));
- assertEquals("1000", worker.getProperty("timeout"));
- assertEquals(2, worker.getFiles().size());
-
- assertTrue(files.contains(worker.getFiles().get(0)));
- assertTrue(files.contains(worker.getFiles().get(1)));
- }
-
- @Test
- public void testOverrideExternalConfiguration() throws IOException {
- Application orig = ExampleAppJson.loadResource(EXTERNAL_JSON_1);
-
- Configuration global = orig.getConfiguration();
- assertEquals(0, global.getProperties().size());
-
- assertEquals(3, orig.getComponents().size());
-
- Configuration simple = orig.getComponent("simple").getConfiguration();
- assertEquals(0, simple.getProperties().size());
-
- Configuration master = orig.getComponent("master").getConfiguration();
- assertEquals(1, master.getProperties().size());
- assertEquals("is-overridden", master.getProperty("g3"));
-
- Configuration other = orig.getComponent("other").getConfiguration();
- assertEquals(0, other.getProperties().size());
-
- // load the external application
- SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
- FileSystem mockFs = createNiceMock(FileSystem.class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- replay(sfs, mockFs);
- Application ext = ExampleAppJson.loadResource(APP_JSON);
- ServiceApiUtil.validateAndResolveApplication(ext, sfs, new
- YarnConfiguration());
- reset(sfs, mockFs);
-
- // perform the resolution on original application
- JsonSerDeser<Application> jsonSerDeser = createNiceMock(JsonSerDeser
- .class);
- expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
- expect(sfs.buildClusterDirPath(anyObject())).andReturn(
- new Path("cluster_dir_path")).anyTimes();
- expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
- .anyTimes();
- replay(sfs, mockFs, jsonSerDeser);
- ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
- ServiceApiUtil.validateAndResolveApplication(orig, sfs, new
- YarnConfiguration());
-
- global = orig.getConfiguration();
- assertEquals(0, global.getProperties().size());
-
- assertEquals(4, orig.getComponents().size());
-
- simple = orig.getComponent("simple").getConfiguration();
- assertEquals(3, simple.getProperties().size());
- assertEquals("a", simple.getProperty("g1"));
- assertEquals("b", simple.getProperty("g2"));
- assertEquals("60",
- simple.getProperty("yarn.service.failure-count-reset.window"));
-
- master = orig.getComponent("master").getConfiguration();
- assertEquals(5, master.getProperties().size());
- assertEquals("512M", master.getProperty("jvm.heapsize"));
- assertEquals("overridden", master.getProperty("g1"));
- assertEquals("b", master.getProperty("g2"));
- assertEquals("is-overridden", master.getProperty("g3"));
- assertEquals("60",
- simple.getProperty("yarn.service.failure-count-reset.window"));
-
- Configuration worker = orig.getComponent("worker").getConfiguration();
- LOG.info("worker = {}", worker);
- assertEquals(4, worker.getProperties().size());
- assertEquals("512M", worker.getProperty("jvm.heapsize"));
- assertEquals("overridden-by-worker", worker.getProperty("g1"));
- assertEquals("b", worker.getProperty("g2"));
- assertEquals("60",
- worker.getProperty("yarn.service.failure-count-reset.window"));
-
- other = orig.getComponent("other").getConfiguration();
- assertEquals(0, other.getProperties().size());
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org